From bec205bbd51271c4a5335f257d2d66e012ddaa39 Mon Sep 17 00:00:00 2001 From: Ulrik Johansson Date: Sun, 9 Sep 2018 13:35:12 +0200 Subject: [PATCH 001/218] Add ability to override the pyup.io API key for the check command --- docs/advanced.rst | 3 +++ pipenv/core.py | 7 ++++--- pipenv/environments.py | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index f3122724..a3f044cb 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -240,6 +240,9 @@ Example:: API access throttling based on overall usage rather than individual client usage. + You can also use your own safety API key by setting the + environment variable ``PIPENV_CHECK_KEY``. + ☤ Community Integrations ------------------------ diff --git a/pipenv/core.py b/pipenv/core.py index 52c127b1..830834a4 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -55,6 +55,7 @@ from .environments import ( PIPENV_DONT_USE_PYENV, SESSION_IS_INTERACTIVE, PIPENV_CACHE_DIR, + PIPENV_CHECK_KEY, ) # Packages that should be ignored later. @@ -2275,15 +2276,15 @@ def do_check( else: ignored = "" c = delegator.run( - '"{0}" {1} check --json --key=1ab8d58f-5122e025-83674263-bc1e79e0 {2}'.format( - python, escape_grouped_arguments(path), ignored + '"{0}" {1} check --json --key={2} {3}'.format( + python, escape_grouped_arguments(path), PIPENV_CHECK_KEY, ignored ) ) try: results = simplejson.loads(c.out) except ValueError: click.echo("An error occurred:", err=True) - click.echo(c.err, err=True) + click.echo(c.err if len(c.err) > 0 else c.out, err=True) sys.exit(1) for (package, resolved, installed, description, vuln) in results: click.echo( diff --git a/pipenv/environments.py b/pipenv/environments.py index b6633f39..6762709d 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -194,6 +194,7 @@ Default is to prompt the user for an answer if the current command line session if interactive. """ +PIPENV_CHECK_KEY = os.environ.get("PIPENV_CHECK_KEY", "1ab8d58f-5122e025-83674263-bc1e79e0") # Internal, support running in a different Python from sys.executable. PIPENV_PYTHON = os.environ.get("PIPENV_PYTHON") From 59824c3725756e5e01508afbb3a01d7247fe2e55 Mon Sep 17 00:00:00 2001 From: Ulrik Johansson Date: Mon, 10 Sep 2018 06:58:55 +0200 Subject: [PATCH 002/218] Add news fragment --- news/2825.feature | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/2825.feature diff --git a/news/2825.feature b/news/2825.feature new file mode 100644 index 00000000..9dcbf4ff --- /dev/null +++ b/news/2825.feature @@ -0,0 +1,2 @@ +Added environment variable `PIPENV_CHECK_KEY` to add ability +to override the bundled pyup.io API key. From ba61dd06220c6e91fc196324745d28a8e4338eb4 Mon Sep 17 00:00:00 2001 From: Ulrik Johansson Date: Wed, 19 Sep 2018 07:40:05 +0200 Subject: [PATCH 003/218] Use less ambiguous env var name for pyup api key --- docs/advanced.rst | 2 +- news/2825.feature | 2 +- pipenv/core.py | 4 ++-- pipenv/environments.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index a3f044cb..f2fdee05 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -241,7 +241,7 @@ Example:: client usage. You can also use your own safety API key by setting the - environment variable ``PIPENV_CHECK_KEY``. + environment variable ``PIPENV_PYUP_API_KEY``. ☤ Community Integrations diff --git a/news/2825.feature b/news/2825.feature index 9dcbf4ff..ba112a58 100644 --- a/news/2825.feature +++ b/news/2825.feature @@ -1,2 +1,2 @@ -Added environment variable `PIPENV_CHECK_KEY` to add ability +Added environment variable `PIPENV_PYUP_API_KEY` to add ability to override the bundled pyup.io API key. diff --git a/pipenv/core.py b/pipenv/core.py index 830834a4..91633aed 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -55,7 +55,7 @@ from .environments import ( PIPENV_DONT_USE_PYENV, SESSION_IS_INTERACTIVE, PIPENV_CACHE_DIR, - PIPENV_CHECK_KEY, + PIPENV_PYUP_API_KEY, ) # Packages that should be ignored later. @@ -2277,7 +2277,7 @@ def do_check( ignored = "" c = delegator.run( '"{0}" {1} check --json --key={2} {3}'.format( - python, escape_grouped_arguments(path), PIPENV_CHECK_KEY, ignored + python, escape_grouped_arguments(path), PIPENV_PYUP_API_KEY, ignored ) ) try: diff --git a/pipenv/environments.py b/pipenv/environments.py index 6762709d..558bafd2 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -194,7 +194,7 @@ Default is to prompt the user for an answer if the current command line session if interactive. """ -PIPENV_CHECK_KEY = os.environ.get("PIPENV_CHECK_KEY", "1ab8d58f-5122e025-83674263-bc1e79e0") +PIPENV_PYUP_API_KEY = os.environ.get("PIPENV_PYUP_API_KEY", "1ab8d58f-5122e025-83674263-bc1e79e0") # Internal, support running in a different Python from sys.executable. PIPENV_PYTHON = os.environ.get("PIPENV_PYTHON") From 63f200765e679aeeb418c1a34822f8edea1abf36 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Fri, 26 Oct 2018 16:47:54 +0800 Subject: [PATCH 004/218] Proper case the path on Windows --- pipenv/project.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index c2dc9668..efb06f10 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -4,6 +4,7 @@ import json import os import re import sys +import glob import base64 import fnmatch import hashlib @@ -49,14 +50,18 @@ def _normalized(p): if p is None: return None loc = vistir.compat.Path(p) - if loc.is_absolute(): - return normalize_drive(str(loc)) - else: + if not loc.is_absolute(): try: loc = loc.resolve() except OSError: loc = loc.absolute() - return normalize_drive(str(loc)) + # From https://stackoverflow.com/a/35229734/5043728 + if os.name == 'nt': + matches = glob.glob(re.sub(r'([^:/\\])(?=[/\\]|$)', r'[\1]', str(loc))) + path_str = matches and matches[0] or str(loc) + else: + path_str = str(loc) + return normalize_drive(path_str) DEFAULT_NEWLINES = u"\n" From bb85253bb39ebc249fb1fe7c6b6138a106a27170 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Fri, 26 Oct 2018 16:52:35 +0800 Subject: [PATCH 005/218] Add a news entry --- news/3104.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/3104.bugfix diff --git a/news/3104.bugfix b/news/3104.bugfix new file mode 100644 index 00000000..1915494d --- /dev/null +++ b/news/3104.bugfix @@ -0,0 +1 @@ +Fix the path casing issue that makes `pipenv clean` fail on Windows From 14be3db996bd2346722c2a2332938f3a4c8b0f8d Mon Sep 17 00:00:00 2001 From: frostming Date: Fri, 26 Oct 2018 17:22:41 +0800 Subject: [PATCH 006/218] Add test case for #3105 --- tests/integration/test_windows.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/integration/test_windows.py b/tests/integration/test_windows.py index 128a6b5d..da138f7a 100644 --- a/tests/integration/test_windows.py +++ b/tests/integration/test_windows.py @@ -65,3 +65,16 @@ def test_local_path_windows_forward_slash(PipenvInstance, pypi): with PipenvInstance(pypi=pypi, chdir=True) as p: c = p.pipenv('install "{0}"'.format(whl.as_posix())) assert c.return_code == 0 + + +@pytest.mark.cli +def test_pipenv_clean_windows(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + c = p.pipenv('install requests') + assert c.return_code == 0 + c = p.pipenv('run pip install click') + assert c.return_code == 0 + + c = p.pipenv('clean --dry-run') + assert c.return_code == 0 + assert 'click' in c.out.strip() From 80a6dcbedde656eb708290b42798b3d3351694e2 Mon Sep 17 00:00:00 2001 From: devxpy Date: Sat, 27 Oct 2018 23:55:34 +0530 Subject: [PATCH 007/218] add PIPENV_SKIP_LOCK envvar, and suitable doc --- pipenv/cli/options.py | 1 + pipenv/environments.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 99fc5344..940bbf77 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -127,6 +127,7 @@ def skip_lock_option(f): return value return option("--skip-lock", is_flag=True, default=False, expose_value=False, help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", + envvar="PIPENV_SKIP_LOCK", callback=callback, type=click_booltype)(f) diff --git a/pipenv/environments.py b/pipenv/environments.py index 4bfd937d..45d5f720 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -197,6 +197,16 @@ Default is to prompt the user for an answer if the current command line session if interactive. """ +PIPENV_SKIP_LOCK = False +"""If set, Pipenv won't lock dependencies automatically. + +This might be desirable if a project has large number of dependencies, +because locking is an inherently slow operation. + +Default is to lock dependencies and update ``Pipfile.lock`` on each run. + +NOTE: This only affects the ``install`` and ``uninstall`` commands. +""" # Internal, support running in a different Python from sys.executable. PIPENV_PYTHON = os.environ.get("PIPENV_PYTHON") From 023e7861c3ed1369fba84ff9ba9b70424251c2b3 Mon Sep 17 00:00:00 2001 From: devxpy Date: Sun, 28 Oct 2018 00:36:45 +0530 Subject: [PATCH 008/218] fix PIPENV_SKIP_LOCK for `pipenv unisntall` command. --- pipenv/cli/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 1ce9fee9..d2f288d3 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -284,7 +284,7 @@ def uninstall( three=state.three, python=state.python, system=state.system, - lock=lock, + lock=not state.installstate.skip_lock, all_dev=all_dev, all=all, keep_outdated=state.installstate.keep_outdated, From a9987d31bb6a3b58251a769bd523dd63f8f2b343 Mon Sep 17 00:00:00 2001 From: frostming Date: Mon, 29 Oct 2018 14:11:25 +0800 Subject: [PATCH 009/218] Comment on what the code is doing --- pipenv/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/project.py b/pipenv/project.py index efb06f10..b8ea464c 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -55,7 +55,7 @@ def _normalized(p): loc = loc.resolve() except OSError: loc = loc.absolute() - # From https://stackoverflow.com/a/35229734/5043728 + # Recase the path properly on Windows. From https://stackoverflow.com/a/35229734/5043728 if os.name == 'nt': matches = glob.glob(re.sub(r'([^:/\\])(?=[/\\]|$)', r'[\1]', str(loc))) path_str = matches and matches[0] or str(loc) From ef4be482a68af578a48513a68723a584a30b9606 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 30 Oct 2018 01:03:07 -0400 Subject: [PATCH 010/218] conditional builds Signed-off-by: Dan Ryan --- .vsts-ci/linux.yml | 19 +++++++++++++++++++ .vsts-ci/windows.yml | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 7968801c..95c62dc5 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -1,3 +1,22 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + phases: - template: phases/test.yml parameters: diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index a397a23c..e423c3ab 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -1,3 +1,22 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + phases: - template: phases/test.yml parameters: From baef2e78a632168bddce0a428969b9f6fb32693c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 19 Oct 2018 20:57:20 -0400 Subject: [PATCH 011/218] Upgrade vendored dependencies - Upgrade pythonfinder - Upgrade vistir - Upgrade requirementslib - Vendor backported version of `functools.lru_cache` for performance - Fix editable dependency installation when markers are present - Fix extraneous resource warnings - Fix filesystem output stream encoding issues - Fix pythonfinder non-standard python name issues - Provide full interaction layer to `Pipfile` and `Pipfile.lock` in requirementslib - Fixes #3017 - Fixes #3014 - Fixes #3021 - Fixes #3019 Signed-off-by: Dan Ryan Update vendored dependencies - Update shellingham, tomlkit, requests, urllib3, certifi, vistir and parse - Fixes #2820 - Fixes #2852 Signed-off-by: Dan Ryan Cleanup old vendored dependencies Signed-off-by: Dan Ryan --- pipenv/environments.py | 17 +- pipenv/resolver.py | 1 - .../backports/functools_lru_cache.LICENSE | 7 + .../vendor/backports/functools_lru_cache.py | 184 +++ pipenv/vendor/certifi/__init__.py | 2 +- pipenv/vendor/certifi/cacert.pem | 30 - pipenv/vendor/cursor/LICENSE | 5 + pipenv/vendor/cursor/__init__.py | 4 + pipenv/vendor/cursor/cursor.py | 57 + pipenv/vendor/parse.py | 33 +- pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/cli.py | 18 +- pipenv/vendor/pythonfinder/environment.py | 3 + pipenv/vendor/pythonfinder/models/mixins.py | 38 +- pipenv/vendor/pythonfinder/models/path.py | 279 +++-- pipenv/vendor/pythonfinder/models/pyenv.py | 204 +++- pipenv/vendor/pythonfinder/models/python.py | 77 +- pipenv/vendor/pythonfinder/models/windows.py | 29 +- pipenv/vendor/pythonfinder/pythonfinder.py | 34 +- pipenv/vendor/pythonfinder/utils.py | 40 +- pipenv/vendor/requirementslib/__init__.py | 2 +- .../vendor/requirementslib/models/lockfile.py | 160 ++- .../vendor/requirementslib/models/pipfile.py | 148 ++- .../vendor/requirementslib/models/project.py | 241 ++++ .../requirementslib/models/requirements.py | 27 +- pipenv/vendor/requirementslib/models/utils.py | 12 + pipenv/vendor/shellingham/__init__.py | 2 +- pipenv/vendor/shellingham/posix/ps.py | 12 +- pipenv/vendor/toml.LICENSE | 26 - pipenv/vendor/toml.py | 1039 ----------------- pipenv/vendor/tomlkit/__init__.py | 2 +- pipenv/vendor/tomlkit/container.py | 19 +- pipenv/vendor/tomlkit/exceptions.py | 2 +- pipenv/vendor/tomlkit/items.py | 129 +- pipenv/vendor/tomlkit/parser.py | 245 ++-- pipenv/vendor/tomlkit/toml_char.py | 12 + pipenv/vendor/urllib3/__init__.py | 9 +- pipenv/vendor/urllib3/_collections.py | 5 +- pipenv/vendor/urllib3/connection.py | 40 +- pipenv/vendor/urllib3/connectionpool.py | 18 +- pipenv/vendor/urllib3/contrib/__init__.py | 0 .../urllib3/contrib/_appengine_environ.py | 30 + .../contrib/_securetransport/__init__.py | 0 .../contrib/_securetransport/bindings.py | 0 .../contrib/_securetransport/low_level.py | 0 pipenv/vendor/urllib3/contrib/appengine.py | 36 +- pipenv/vendor/urllib3/contrib/ntlmpool.py | 3 +- pipenv/vendor/urllib3/contrib/pyopenssl.py | 25 +- .../vendor/urllib3/contrib/securetransport.py | 0 pipenv/vendor/urllib3/contrib/socks.py | 0 pipenv/vendor/urllib3/exceptions.py | 0 pipenv/vendor/urllib3/fields.py | 0 pipenv/vendor/urllib3/filepost.py | 0 pipenv/vendor/urllib3/packages/__init__.py | 0 .../urllib3/packages/backports/__init__.py | 0 .../urllib3/packages/backports/makefile.py | 2 +- .../vendor/urllib3/packages/ordered_dict.py | 259 ---- pipenv/vendor/urllib3/packages/six.py | 0 .../packages/ssl_match_hostname/__init__.py | 0 .../ssl_match_hostname/_implementation.py | 3 +- pipenv/vendor/urllib3/poolmanager.py | 1 + pipenv/vendor/urllib3/request.py | 2 +- pipenv/vendor/urllib3/response.py | 41 +- pipenv/vendor/urllib3/util/__init__.py | 0 pipenv/vendor/urllib3/util/connection.py | 8 + pipenv/vendor/urllib3/util/queue.py | 0 pipenv/vendor/urllib3/util/request.py | 0 pipenv/vendor/urllib3/util/response.py | 10 +- pipenv/vendor/urllib3/util/retry.py | 2 +- pipenv/vendor/urllib3/util/ssl_.py | 33 +- pipenv/vendor/urllib3/util/timeout.py | 0 pipenv/vendor/urllib3/util/url.py | 0 pipenv/vendor/urllib3/util/wait.py | 5 +- pipenv/vendor/vendor.txt | 18 +- pipenv/vendor/vistir/__init__.py | 10 +- pipenv/vendor/vistir/compat.py | 18 +- pipenv/vendor/vistir/contextmanagers.py | 70 +- pipenv/vendor/vistir/misc.py | 129 +- pipenv/vendor/vistir/path.py | 77 +- pipenv/vendor/vistir/spin.py | 149 +++ pipenv/vendor/vistir/termcolors.py | 116 ++ 81 files changed, 2241 insertions(+), 2020 deletions(-) create mode 100644 pipenv/vendor/backports/functools_lru_cache.LICENSE create mode 100644 pipenv/vendor/backports/functools_lru_cache.py create mode 100644 pipenv/vendor/cursor/LICENSE create mode 100644 pipenv/vendor/cursor/__init__.py create mode 100644 pipenv/vendor/cursor/cursor.py create mode 100644 pipenv/vendor/requirementslib/models/project.py delete mode 100644 pipenv/vendor/toml.LICENSE delete mode 100644 pipenv/vendor/toml.py mode change 100755 => 100644 pipenv/vendor/urllib3/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/_collections.py mode change 100755 => 100644 pipenv/vendor/urllib3/connection.py mode change 100755 => 100644 pipenv/vendor/urllib3/connectionpool.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/__init__.py create mode 100644 pipenv/vendor/urllib3/contrib/_appengine_environ.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/_securetransport/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/_securetransport/bindings.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/_securetransport/low_level.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/appengine.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/ntlmpool.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/pyopenssl.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/securetransport.py mode change 100755 => 100644 pipenv/vendor/urllib3/contrib/socks.py mode change 100755 => 100644 pipenv/vendor/urllib3/exceptions.py mode change 100755 => 100644 pipenv/vendor/urllib3/fields.py mode change 100755 => 100644 pipenv/vendor/urllib3/filepost.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/backports/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/backports/makefile.py delete mode 100755 pipenv/vendor/urllib3/packages/ordered_dict.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/six.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py mode change 100755 => 100644 pipenv/vendor/urllib3/poolmanager.py mode change 100755 => 100644 pipenv/vendor/urllib3/request.py mode change 100755 => 100644 pipenv/vendor/urllib3/response.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/__init__.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/connection.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/queue.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/request.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/response.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/retry.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/ssl_.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/timeout.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/url.py mode change 100755 => 100644 pipenv/vendor/urllib3/util/wait.py create mode 100644 pipenv/vendor/vistir/spin.py create mode 100644 pipenv/vendor/vistir/termcolors.py diff --git a/pipenv/environments.py b/pipenv/environments.py index 4bfd937d..e6de4f0d 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -8,6 +8,8 @@ from .vendor.vistir.misc import fs_str # I hope I can remove this one day. os.environ["PYTHONDONTWRITEBYTECODE"] = fs_str("1") +PIPENV_IS_CI = bool("CI" in os.environ or "TF_BUILD" in os.environ) + # HACK: Prevent invalid shebangs with Homebrew-installed Python: # https://bugs.python.org/issue22490 os.environ.pop("__PYVENV_LAUNCHER__", None) @@ -68,7 +70,7 @@ PIPENV_HIDE_EMOJIS = bool(os.environ.get("PIPENV_HIDE_EMOJIS")) Default is to show emojis. This is automatically set on Windows. """ -if os.name == "nt": +if os.name == "nt" or PIPENV_IS_CI: PIPENV_HIDE_EMOJIS = True PIPENV_IGNORE_VIRTUALENVS = bool(os.environ.get("PIPENV_IGNORE_VIRTUALENVS")) @@ -94,7 +96,7 @@ Default is 3. See also ``PIPENV_NO_INHERIT``. PIPENV_MAX_RETRIES = int(os.environ.get( "PIPENV_MAX_RETRIES", - "1" if "CI" in os.environ else "0", + "1" if PIPENV_IS_CI else "0", )) """Specify how many retries Pipenv should attempt for network requests. @@ -128,9 +130,18 @@ PIPENV_NOSPIN = bool(os.environ.get("PIPENV_NOSPIN")) This can make the logs cleaner. Automatically set on Windows, and in CI environments. """ -if os.name == "nt" or "CI" in os.environ: +if PIPENV_IS_CI: PIPENV_NOSPIN = True +PIPENV_SPINNER = "dots" +"""Sets the default spinner type. + +Spinners are identitcal to the node.js spinners and can be found at +https://github.com/sindresorhus/cli-spinners +""" +if os.name == "nt": + PIPENV_SPINNER = "bouncingBar" + PIPENV_PIPFILE = os.environ.get("PIPENV_PIPFILE") """If set, this specifies a custom Pipfile location. diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 6526d990..8c282b71 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -80,7 +80,6 @@ def main(): if pypi_mirror_source else project.pipfile_sources ) - print("using sources: %s" % sources) results = resolve( packages, pre=do_pre, diff --git a/pipenv/vendor/backports/functools_lru_cache.LICENSE b/pipenv/vendor/backports/functools_lru_cache.LICENSE new file mode 100644 index 00000000..5e795a61 --- /dev/null +++ b/pipenv/vendor/backports/functools_lru_cache.LICENSE @@ -0,0 +1,7 @@ +Copyright Jason R. Coombs + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pipenv/vendor/backports/functools_lru_cache.py b/pipenv/vendor/backports/functools_lru_cache.py new file mode 100644 index 00000000..707c6c76 --- /dev/null +++ b/pipenv/vendor/backports/functools_lru_cache.py @@ -0,0 +1,184 @@ +from __future__ import absolute_import + +import functools +from collections import namedtuple +from threading import RLock + +_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) + + +@functools.wraps(functools.update_wrapper) +def update_wrapper(wrapper, + wrapped, + assigned = functools.WRAPPER_ASSIGNMENTS, + updated = functools.WRAPPER_UPDATES): + """ + Patch two bugs in functools.update_wrapper. + """ + # workaround for http://bugs.python.org/issue3445 + assigned = tuple(attr for attr in assigned if hasattr(wrapped, attr)) + wrapper = functools.update_wrapper(wrapper, wrapped, assigned, updated) + # workaround for https://bugs.python.org/issue17482 + wrapper.__wrapped__ = wrapped + return wrapper + + +class _HashedSeq(list): + __slots__ = 'hashvalue' + + def __init__(self, tup, hash=hash): + self[:] = tup + self.hashvalue = hash(tup) + + def __hash__(self): + return self.hashvalue + + +def _make_key(args, kwds, typed, + kwd_mark=(object(),), + fasttypes=set([int, str, frozenset, type(None)]), + sorted=sorted, tuple=tuple, type=type, len=len): + 'Make a cache key from optionally typed positional and keyword arguments' + key = args + if kwds: + sorted_items = sorted(kwds.items()) + key += kwd_mark + for item in sorted_items: + key += item + if typed: + key += tuple(type(v) for v in args) + if kwds: + key += tuple(type(v) for k, v in sorted_items) + elif len(key) == 1 and type(key[0]) in fasttypes: + return key[0] + return _HashedSeq(key) + + +def lru_cache(maxsize=100, typed=False): + """Least-recently-used cache decorator. + + If *maxsize* is set to None, the LRU features are disabled and the cache + can grow without bound. + + If *typed* is True, arguments of different types will be cached separately. + For example, f(3.0) and f(3) will be treated as distinct calls with + distinct results. + + Arguments to the cached function must be hashable. + + View the cache statistics named tuple (hits, misses, maxsize, currsize) with + f.cache_info(). Clear the cache and statistics with f.cache_clear(). + Access the underlying function with f.__wrapped__. + + See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used + + """ + + # Users should only access the lru_cache through its public API: + # cache_info, cache_clear, and f.__wrapped__ + # The internals of the lru_cache are encapsulated for thread safety and + # to allow the implementation to change (including a possible C version). + + def decorating_function(user_function): + + cache = dict() + stats = [0, 0] # make statistics updateable non-locally + HITS, MISSES = 0, 1 # names for the stats fields + make_key = _make_key + cache_get = cache.get # bound method to lookup key or return None + _len = len # localize the global len() function + lock = RLock() # because linkedlist updates aren't threadsafe + root = [] # root of the circular doubly linked list + root[:] = [root, root, None, None] # initialize by pointing to self + nonlocal_root = [root] # make updateable non-locally + PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields + + if maxsize == 0: + + def wrapper(*args, **kwds): + # no caching, just do a statistics update after a successful call + result = user_function(*args, **kwds) + stats[MISSES] += 1 + return result + + elif maxsize is None: + + def wrapper(*args, **kwds): + # simple caching without ordering or size limit + key = make_key(args, kwds, typed) + result = cache_get(key, root) # root used here as a unique not-found sentinel + if result is not root: + stats[HITS] += 1 + return result + result = user_function(*args, **kwds) + cache[key] = result + stats[MISSES] += 1 + return result + + else: + + def wrapper(*args, **kwds): + # size limited caching that tracks accesses by recency + key = make_key(args, kwds, typed) if kwds or typed else args + with lock: + link = cache_get(key) + if link is not None: + # record recent use of the key by moving it to the front of the list + root, = nonlocal_root + link_prev, link_next, key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + last = root[PREV] + last[NEXT] = root[PREV] = link + link[PREV] = last + link[NEXT] = root + stats[HITS] += 1 + return result + result = user_function(*args, **kwds) + with lock: + root, = nonlocal_root + if key in cache: + # getting here means that this same key was added to the + # cache while the lock was released. since the link + # update is already done, we need only return the + # computed result and update the count of misses. + pass + elif _len(cache) >= maxsize: + # use the old root to store the new key and result + oldroot = root + oldroot[KEY] = key + oldroot[RESULT] = result + # empty the oldest link and make it the new root + root = nonlocal_root[0] = oldroot[NEXT] + oldkey = root[KEY] + root[KEY] = root[RESULT] = None + # now update the cache dictionary for the new links + del cache[oldkey] + cache[key] = oldroot + else: + # put result in a new link at the front of the list + last = root[PREV] + link = [last, root, key, result] + last[NEXT] = root[PREV] = cache[key] = link + stats[MISSES] += 1 + return result + + def cache_info(): + """Report cache statistics""" + with lock: + return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache)) + + def cache_clear(): + """Clear the cache and cache statistics""" + with lock: + cache.clear() + root = nonlocal_root[0] + root[:] = [root, root, None, None] + stats[:] = [0, 0] + + wrapper.__wrapped__ = user_function + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return update_wrapper(wrapper, user_function) + + return decorating_function diff --git a/pipenv/vendor/certifi/__init__.py b/pipenv/vendor/certifi/__init__.py index aa329fbb..50f2e130 100644 --- a/pipenv/vendor/certifi/__init__.py +++ b/pipenv/vendor/certifi/__init__.py @@ -1,3 +1,3 @@ from .core import where, old_where -__version__ = "2018.08.24" +__version__ = "2018.10.15" diff --git a/pipenv/vendor/certifi/cacert.pem b/pipenv/vendor/certifi/cacert.pem index 85de024e..e75d85b3 100644 --- a/pipenv/vendor/certifi/cacert.pem +++ b/pipenv/vendor/certifi/cacert.pem @@ -326,36 +326,6 @@ OCiNUW7dFGdTbHFcJoRNdVq2fmBWqU2t+5sel/MN2dKXVHfaPRK34B7vCAas+YWH QMAJKOSLakhT2+zNVVXxxvjpoixMptEmX36vWkzaH6byHCx+rgIW0lbQL1dTR+iS -----END CERTIFICATE----- -# Issuer: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Subject: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Label: "Visa eCommerce Root" -# Serial: 25952180776285836048024890241505565794 -# MD5 Fingerprint: fc:11:b8:d8:08:93:30:00:6d:23:f9:7e:eb:52:1e:02 -# SHA1 Fingerprint: 70:17:9b:86:8c:00:a4:fa:60:91:52:22:3f:9f:3e:32:bd:e0:05:62 -# SHA256 Fingerprint: 69:fa:c9:bd:55:fb:0a:c7:8d:53:bb:ee:5c:f1:d5:97:98:9f:d0:aa:ab:20:a2:51:51:bd:f1:73:3e:e7:d1:22 ------BEGIN CERTIFICATE----- -MIIDojCCAoqgAwIBAgIQE4Y1TR0/BvLB+WUF1ZAcYjANBgkqhkiG9w0BAQUFADBr -MQswCQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRl -cm5hdGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNv -bW1lcmNlIFJvb3QwHhcNMDIwNjI2MDIxODM2WhcNMjIwNjI0MDAxNjEyWjBrMQsw -CQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRlcm5h -dGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNvbW1l -cmNlIFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvV95WHm6h -2mCxlCfLF9sHP4CFT8icttD0b0/Pmdjh28JIXDqsOTPHH2qLJj0rNfVIsZHBAk4E -lpF7sDPwsRROEW+1QK8bRaVK7362rPKgH1g/EkZgPI2h4H3PVz4zHvtH8aoVlwdV -ZqW1LS7YgFmypw23RuwhY/81q6UCzyr0TP579ZRdhE2o8mCP2w4lPJ9zcc+U30rq -299yOIzzlr3xF7zSujtFWsan9sYXiwGd/BmoKoMWuDpI/k4+oKsGGelT84ATB+0t -vz8KPFUgOSwsAGl0lUq8ILKpeeUYiZGo3BxN77t+Nwtd/jmliFKMAGzsGHxBvfaL -dXe6YJ2E5/4tAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD -AgEGMB0GA1UdDgQWBBQVOIMPPyw/cDMezUb+B4wg4NfDtzANBgkqhkiG9w0BAQUF -AAOCAQEAX/FBfXxcCLkr4NWSR/pnXKUTwwMhmytMiUbPWU3J/qVAtmPN3XEolWcR -zCSs00Rsca4BIGsDoo8Ytyk6feUWYFN4PMCvFYP3j1IzJL1kk5fui/fbGKhtcbP3 -LBfQdCVp9/5rPJS+TUtBjE7ic9DjkCJzQ83z7+pzzkWKsKZJ/0x9nXGIxHYdkFsd -7v3M9+79YKWxehZx0RbQfBI8bGmX265fOZpwLwU8GUYEmSA20GBuYQa7FkKMcPcw -++DbZqMAAb3mLNqRX6BGi01qnD093QVG/na/oAo85ADmJ7f/hC3euiInlhBx6yLt -398znM/jra6O1I7mT1GvFpLgXPYHDw== ------END CERTIFICATE----- - # Issuer: CN=AAA Certificate Services O=Comodo CA Limited # Subject: CN=AAA Certificate Services O=Comodo CA Limited # Label: "Comodo AAA Services root" diff --git a/pipenv/vendor/cursor/LICENSE b/pipenv/vendor/cursor/LICENSE new file mode 100644 index 00000000..00023c80 --- /dev/null +++ b/pipenv/vendor/cursor/LICENSE @@ -0,0 +1,5 @@ +This work is licensed under the Creative Commons +Attribution-ShareAlike 2.5 International License. To view a copy of +this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or +send a letter to Creative Commons, PO Box 1866, Mountain View, +CA 94042, USA. diff --git a/pipenv/vendor/cursor/__init__.py b/pipenv/vendor/cursor/__init__.py new file mode 100644 index 00000000..76a4f671 --- /dev/null +++ b/pipenv/vendor/cursor/__init__.py @@ -0,0 +1,4 @@ +from .cursor import hide, show, HiddenCursor + +__all__ = ["hide", "show", "HiddenCursor"] + diff --git a/pipenv/vendor/cursor/cursor.py b/pipenv/vendor/cursor/cursor.py new file mode 100644 index 00000000..e4407c02 --- /dev/null +++ b/pipenv/vendor/cursor/cursor.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +## Author: James Spencer: http://stackoverflow.com/users/1375885/james-spencer +## Packager: Gijs TImmers: https://github.com/GijsTimmers + +## Based on James Spencer's answer on StackOverflow: +## http://stackoverflow.com/questions/5174810/how-to-turn-off-blinking-cursor-in-command-window + +## Licence: CC-BY-SA-2.5 +## http://creativecommons.org/licenses/by-sa/2.5/ + +## This work is licensed under the Creative Commons +## Attribution-ShareAlike 2.5 International License. To view a copy of +## this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or +## send a letter to Creative Commons, PO Box 1866, Mountain View, +## CA 94042, USA. + +import sys +import os + +if os.name == 'nt': + import ctypes + + class _CursorInfo(ctypes.Structure): + _fields_ = [("size", ctypes.c_int), + ("visible", ctypes.c_byte)] + +def hide(stream=sys.stdout): + if os.name == 'nt': + ci = _CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) + ci.visible = False + ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) + elif os.name == 'posix': + stream.write("\033[?25l") + stream.flush() + +def show(stream=sys.stdout): + if os.name == 'nt': + ci = _CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) + ci.visible = True + ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) + elif os.name == 'posix': + stream.write("\033[?25h") + stream.flush() + +class HiddenCursor(object): + def __init__(self, stream=sys.stdout): + self._stream = stream + def __enter__(self): + hide(stream=self._stream) + def __exit__(self, type, value, traceback): + show(stream=self._stream) \ No newline at end of file diff --git a/pipenv/vendor/parse.py b/pipenv/vendor/parse.py index ba58decb..7f9f0786 100644 --- a/pipenv/vendor/parse.py +++ b/pipenv/vendor/parse.py @@ -186,6 +186,19 @@ And messing about with alignment: Note that the "center" alignment does not test to make sure the value is centered - it just strips leading and trailing whitespace. +Width and precision may be used to restrict the size of matched text +from the input. Width specifies a minimum size and precision specifies +a maximum. For example: + +>>> parse('{:.2}{:.2}', 'look') # specifying precision + +>>> parse('{:4}{:4}', 'look at that') # specifying width + +>>> parse('{:4}{:.4}', 'look at that') # specifying both + +>>> parse('{:2d}{:2d}', '0440') # parsing two contiguous numbers + + Some notes for the date and time types: - the presence of the time part is optional (including ISO 8601, starting @@ -329,6 +342,9 @@ the pattern, the actual match represents the shortest successful match for **Version history (in brief)**: +- 1.9.0 We now honor precision and width specifiers when parsing numbers + and strings, allowing parsing of concatenated elements of fixed width + (thanks Julia Signell) - 1.8.4 Add LICENSE file at request of packagers. Correct handling of AM/PM to follow most common interpretation. Correct parsing of hexadecimal that looks like a binary prefix. @@ -389,7 +405,7 @@ See the end of the source file for the license of use. ''' from __future__ import absolute_import -__version__ = '1.8.4' +__version__ = '1.9.0' # yes, I now have two problems import re @@ -977,7 +993,11 @@ class Parser(object): self._group_index += 2 self._type_conversions[group] = lambda s, m: float(s) elif type == 'd': - s = r'\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|0[oO][0-7]+' + if format.get('width'): + width = '{1,%s}' % int(format['width']) + else: + width = '+' + s = '\\d{w}|0[xX][0-9a-fA-F]{w}|0[bB][01]{w}|0[oO][0-7]{w}'.format(w=width) self._type_conversions[group] = int_convert(10) elif type == 'ti': s = r'(\d{4}-\d\d-\d\d)((\s+|T)%s)?(Z|\s*[-+]\d\d:?\d\d)?' % \ @@ -1038,6 +1058,13 @@ class Parser(object): elif type: s = r'\%s+' % type + elif format.get('precision'): + if format.get('width'): + s = '.{%s,%s}?' % (format['width'], format['precision']) + else: + s = '.{1,%s}?' % format['precision'] + elif format.get('width'): + s = '.{%s,}?' % format['width'] else: s = '.+?' @@ -1053,8 +1080,6 @@ class Parser(object): if not fill: fill = '0' s = '%s*' % fill + s - elif format['zero']: - s = '0*' + s # allow numbers to be prefixed with a sign s = r'[-+ ]?' + s diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 672724b4..9ac6031c 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.2' +__version__ = '1.1.3' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index b5aa7da3..1757c081 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -17,7 +17,7 @@ from .pythonfinder import Finder @click.option( "--version", is_flag=True, default=False, help="Display PythonFinder version." ) -@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, help="Ignore unsupported python versions.") +@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", help="Ignore unsupported python versions.") @click.version_option(prog_name='pyfinder', version=__version__) @click.pass_context def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True): @@ -36,7 +36,7 @@ def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsup for v in versions: py = v.py_version click.secho( - "Python: {py.version!s} ({py.architecture!s}) @ {py.comes_from.path!s}".format( + "{py.name!s}: {py.version!s} ({py.architecture!s}) @ {py.comes_from.path!s}".format( py=py ), fg="yellow", @@ -47,23 +47,21 @@ def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsup fg="red", ) if find: - if any([find.startswith("{0}".format(n)) for n in range(10)]): - found = finder.find_python_version(find.strip()) - else: - found = finder.system_path.python_executables + click.secho("Searching for python: {0!s}".format(find.strip()), fg="yellow") + found = finder.find_python_version(find.strip()) if found: - click.echo("Found Python Version: {0}".format(found), color="white") + click.secho("Found python at the following locations:", fg="green") sys.exit(0) else: - click.echo("Failed to find matching executable...") + click.secho("Failed to find matching executable...", fg="yellow") sys.exit(1) elif which: found = finder.system_path.which(which.strip()) if found: - click.echo("Found Executable: {0}".format(found), color="white") + click.secho("Found Executable: {0}".format(found), fg="white") sys.exit(0) else: - click.echo("Failed to find matching executable...") + click.secho("Failed to find matching executable...", fg="yellow") sys.exit(1) else: click.echo("Please provide a command", color="red") diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index 2cdb5fd9..7c69b9fc 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -15,3 +15,6 @@ if sys.maxsize > 2 ** 32: IS_64BIT_OS = platform.machine() == "AMD64" else: IS_64BIT_OS = False + + +IGNORE_UNSUPPORTED = bool(os.environ.get("PYTHONFINDER_IGNORE_UNSUPPORTED", False)) diff --git a/pipenv/vendor/pythonfinder/models/mixins.py b/pipenv/vendor/pythonfinder/models/mixins.py index 8cbd45df..7d406548 100644 --- a/pipenv/vendor/pythonfinder/models/mixins.py +++ b/pipenv/vendor/pythonfinder/models/mixins.py @@ -2,12 +2,14 @@ from __future__ import absolute_import, unicode_literals import abc +import attr import operator import six -from ..utils import KNOWN_EXTS, unnest +from ..utils import ensure_path, KNOWN_EXTS, unnest +@attr.s class BasePath(object): def which(self, name): """Search in this path for an executable. @@ -33,7 +35,14 @@ class BasePath(object): return found def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. Return all copies @@ -44,6 +53,7 @@ class BasePath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ @@ -52,7 +62,14 @@ class BasePath(object): "find_all_python_versions" if self.is_dir else "find_python_version" ) sub_finder = operator.methodcaller( - call_method, major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + call_method, + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, ) if not self.is_dir: return sub_finder(self) @@ -61,7 +78,14 @@ class BasePath(object): return [c for c in sorted(path_filter, key=version_sort, reverse=True)] def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search or self for the specified Python version and return the first match. @@ -72,6 +96,7 @@ class BasePath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ @@ -83,12 +108,13 @@ class BasePath(object): pre=pre, dev=dev, arch=arch, + name=name, ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") if not self.is_dir: - if self.is_python and self.as_python and version_matcher(self.as_python): - return self + if self.is_python and self.as_python and version_matcher(self.py_version): + return attr.evolve(self) return finder = ( (child, child.as_python) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index c90d9be3..d54393b6 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -10,6 +10,7 @@ from collections import defaultdict from itertools import chain import attr +import six from cached_property import cached_property @@ -19,8 +20,12 @@ from .mixins import BasePath from ..environment import PYENV_INSTALLED, PYENV_ROOT from ..exceptions import InvalidPythonVersion from ..utils import ( - ensure_path, filter_pythons, looks_like_python, optional_instance_of, - path_is_known_executable, unnest + ensure_path, + filter_pythons, + looks_like_python, + optional_instance_of, + path_is_known_executable, + unnest, ) from .python import PythonVersion @@ -75,7 +80,7 @@ class SystemPath(object): if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue - if isinstance(entry, VersionPath): + if type(entry).__name__ == "VersionPath": for path in entry.paths.values(): if path not in self._version_dict[version] and path.is_python: self._version_dict[version].append(path) @@ -130,17 +135,16 @@ class SystemPath(object): pyenv_index = self.path_order.index(last_pyenv) except ValueError: return - self.pyenv_finder = PyenvFinder.create(root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported) - # paths = (v.paths.values() for v in self.pyenv_finder.versions.values()) - root_paths = ( - p for path in self.pyenv_finder.expanded_paths for p in path if p.is_root + self.pyenv_finder = PyenvFinder.create( + root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported ) + root_paths = [p for p in self.pyenv_finder.roots] before_path = self.path_order[: pyenv_index + 1] after_path = self.path_order[pyenv_index + 2 :] self.path_order = ( - before_path + [p.path.as_posix() for p in root_paths] + after_path + before_path + [p.as_posix() for p in root_paths] + after_path ) - self.paths.update({p.path: p for p in root_paths}) + self.paths.update(self.pyenv_finder.roots) self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): @@ -155,7 +159,9 @@ class SystemPath(object): def get_path(self, path): path = ensure_path(path) - _path = self.paths.get(path.as_posix()) + _path = self.paths.get(path) + if not _path: + _path = self.paths.get(path.as_posix()) if not _path and path.as_posix() in self.path_order: _path = PathEntry.create( path=path.absolute(), is_root=True, only_python=self.only_python @@ -163,6 +169,14 @@ class SystemPath(object): self.paths[path.as_posix()] = _path return _path + def _get_paths(self): + return (self.get_path(k) for k in self.path_order) + + @cached_property + def path_entries(self): + paths = self._get_paths() + return paths + def find_all(self, executable): """Search the path for an executable. Return all copies. @@ -171,8 +185,8 @@ class SystemPath(object): :returns: List[PathEntry] """ sub_which = operator.methodcaller("which", name=executable) - filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) - return [f for f in filtered] + filtered = (sub_which(self.get_path(k)) for k in self.path_order) + return list(filtered) def which(self, executable): """Search for an executable on the path. @@ -182,11 +196,39 @@ class SystemPath(object): :returns: :class:`~pythonfinder.models.PathEntry` object. """ sub_which = operator.methodcaller("which", name=executable) - filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) - return next((f for f in filtered), None) + filtered = (sub_which(self.get_path(k)) for k in self.path_order) + return next(iter(f for f in filtered if f is not None), None) + + def _filter_paths(self, finder): + return ( + pth for pth in unnest(finder(p) for p in self.path_entries if p is not None) + if pth is not None + ) + + def _get_all_pythons(self, finder): + paths = {p.path.as_posix(): p for p in self._filter_paths(finder)} + paths.update(self.python_executables) + return (p for p in paths.values() if p is not None) + + def get_pythons(self, finder): + sort_key = operator.attrgetter("as_python.version_sort") + return ( + k for k in sorted( + (p for p in self._filter_paths(finder) if p.is_python), + key=sort_key, + reverse=True + ) if k is not None + ) def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. Return all copies @@ -197,32 +239,46 @@ class SystemPath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ sub_finder = operator.methodcaller( "find_all_python_versions", - major, + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, + name=name, ) + alternate_sub_finder = None + if major and not (minor or patch or pre or dev or arch or name): + alternate_sub_finder = operator.methodcaller( + "find_all_python_versions", + major=None, + name=major + ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version - paths = (self.get_path(k) for k in self.path_order) - path_filter = filter( - None, unnest((sub_finder(p) for p in paths if p is not None)) - ) - version_sort = operator.attrgetter("as_python.version_sort") - return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + values = list(self.get_pythons(sub_finder)) + if not values and alternate_sub_finder is not None: + values = list(self.get_pythons(alternate_sub_finder)) + return values def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. @@ -233,10 +289,24 @@ class SystemPath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. :rtype: :class:`~pythonfinder.models.PathEntry` """ + if isinstance(major, six.string_types) and not minor and not patch: + # Only proceed if this is in the format "x.y.z" or similar + if major.count(".") > 0 and major[0].isdigit(): + version = major.split(".", 2) + if len(version) > 3: + major, minor, patch, rest = version + elif len(version) == 3: + major, minor, patch = version + else: + major, minor = version + else: + name = "{0!s}".format(major) + major = None sub_finder = operator.methodcaller( "find_python_version", major, @@ -245,7 +315,15 @@ class SystemPath(object): pre=pre, dev=dev, arch=arch, + name=name, ) + alternate_sub_finder = None + if major and not (minor or patch or pre or dev or arch or name): + alternate_sub_finder = operator.methodcaller( + "find_all_python_versions", + major=None, + name=major + ) if major and minor and patch: _tuple_pre = pre if pre is not None else False _tuple_dev = dev if dev is not None else False @@ -255,37 +333,41 @@ class SystemPath(object): windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version - paths = (self.get_path(k) for k in self.path_order) - path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) - version_sort = operator.attrgetter("as_python.version_sort") - ver = next( - (c for c in sorted(path_filter, key=version_sort, reverse=True)), None - ) + ver = next(iter(self.get_pythons(sub_finder)), None) + if not ver and alternate_sub_finder is not None: + ver = next(iter(self.get_pythons(alternate_sub_finder)), None) if ver: if ver.as_python.version_tuple[:5] in self.python_version_dict: self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) else: self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver] + print(ver) return ver @classmethod - def create(cls, path=None, system=False, only_python=False, global_search=True, ignore_unsupported=False): + def create( + cls, + path=None, + system=False, + only_python=False, + global_search=True, + ignore_unsupported=True, + ): """Create a new :class:`pythonfinder.models.SystemPath` instance. :param path: Search path to prepend when searching, defaults to None :param path: str, optional - :param system: Whether to use the running python by default instead of searching, defaults to False - :param system: bool, optional - :param only_python: Whether to search only for python executables, defaults to False - :param only_python: bool, optional - :param ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True - :param ignore_unsupported: bool, optional + :param bool system: Whether to use the running python by default instead of searching, defaults to False + :param bool only_python: Whether to search only for python executables, defaults to False + :param bool ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True :return: A new :class:`pythonfinder.models.SystemPath` instance. :rtype: :class:`pythonfinder.models.SystemPath` """ path_entries = defaultdict(PathEntry) paths = [] + if ignore_unsupported: + os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1") if global_search: paths = os.environ.get("PATH").split(os.pathsep) if path: @@ -316,7 +398,8 @@ class PathEntry(BasePath): _children = attr.ib(default=attr.Factory(dict)) is_root = attr.ib(default=True) only_python = attr.ib(default=False) - py_version = attr.ib(default=None) + name = attr.ib() + py_version = attr.ib() pythons = attr.ib() def __str__(self): @@ -329,17 +412,46 @@ class PathEntry(BasePath): children = self.path.iterdir() return children + def _gen_children(self): + pass_name = self.name != self.path.name + pass_args = {"is_root": False, "only_python": self.only_python} + if pass_name: + pass_args["name"] = self.name + + if not self.is_dir: + yield (self.path.as_posix(), copy.deepcopy(self)) + elif self.is_root: + for child in self._filter_children(): + yield (child.as_posix(), PathEntry.create(path=child, **pass_args)) + return + @cached_property def children(self): - if not self._children and self.is_dir and self.is_root: - self._children = { - child.as_posix(): PathEntry.create(path=child, is_root=False) - for child in self._filter_children() - } - elif not self.is_dir: - self._children = {self.path.as_posix(): self} + if not self._children: + children = {} + for child_key, child_val in self._gen_children(): + children[child_key] = child_val + self._children = children return self._children + @name.default + def get_name(self): + return self.path.name + + @py_version.default + def get_py_version(self): + from ..environment import IGNORE_UNSUPPORTED + if self.is_dir: + return None + if self.is_python: + from .python import PythonVersion + try: + py_version = PythonVersion.from_path(path=self, name=self.name) + except InvalidPythonVersion: + py_version = None + return py_version + return + @pythons.default def get_pythons(self): pythons = defaultdict() @@ -351,56 +463,62 @@ class PathEntry(BasePath): else: if self.is_python: _path = ensure_path(self.path) - pythons[_path.as_posix()] = copy.deepcopy(self) + pythons[_path.as_posix()] = self return pythons @cached_property def as_python(self): + py_version = None + if self.py_version: + return self.py_version if not self.is_dir and self.is_python: - if not self.py_version: - try: - from .python import PythonVersion - - self.py_version = PythonVersion.from_path(self.path) - except (ValueError, InvalidPythonVersion): - self.py_version = None - return self.py_version + try: + from .python import PythonVersion + py_version = PythonVersion.from_path(path=attr.evolve(self), name=self.name) + except (ValueError, InvalidPythonVersion): + py_version = None + return py_version @classmethod - def create(cls, path, is_root=False, only_python=False, pythons=None): + def create(cls, path, is_root=False, only_python=False, pythons=None, name=None): """Helper method for creating new :class:`pythonfinder.models.PathEntry` instances. - :param path: Path to the specified location. - :type path: str - :param is_root: Whether this is a root from the environment PATH variable, defaults to False - :param is_root: bool, optional - :param only_python: Whether to search only for python executables, defaults to False - :param only_python: bool, optional - :param pythons: A dictionary of existing python objects (usually from a finder), defaults to None - :param pythons: dict, optional + :param str path: Path to the specified location. + :param bool is_root: Whether this is a root from the environment PATH variable, defaults to False + :param bool only_python: Whether to search only for python executables, defaults to False + :param dict pythons: A dictionary of existing python objects (usually from a finder), defaults to None + :param str name: Name of the python version, e.g. ``anaconda3-5.3.0`` :return: A new instance of the class. :rtype: :class:`pythonfinder.models.PathEntry` """ target = ensure_path(path) - creation_args = {"path": target, "is_root": is_root, "only_python": only_python} + guessed_name = False + if not name: + guessed_name = True + name = target.name + creation_args = {"path": target, "is_root": is_root, "only_python": only_python, "name": name} if pythons: creation_args["pythons"] = pythons _new = cls(**creation_args) if pythons and only_python: children = {} + child_creation_args = { + "is_root": False, + "py_version": python, + "only_python": only_python + } + if not guessed_name: + child_creation_args["name"] = name for pth, python in pythons.items(): pth = ensure_path(pth) children[pth.as_posix()] = PathEntry( - path=pth, is_root=False, only_python=only_python, py_version=python + path=pth, + **child_creation_args ) _new._children = children return _new - @cached_property - def name(self): - return self.path.name - @cached_property def is_dir(self): try: @@ -416,28 +534,5 @@ class PathEntry(BasePath): @cached_property def is_python(self): return self.is_executable and ( - self.py_version or looks_like_python(self.path.name) + looks_like_python(self.path.name) ) - - -@attr.s -class VersionPath(SystemPath): - base = attr.ib(default=None, validator=optional_instance_of(Path)) - - @classmethod - def create(cls, path, only_python=True, pythons=None): - """Accepts a path to a base python version directory. - - Generates the pyenv version listings for it""" - path = ensure_path(path) - path_entries = defaultdict(PathEntry) - if not path.name.lower() in ["scripts", "bin"]: - bin_name = "Scripts" if os.name == "nt" else "bin" - bin_dir = path / bin_name - else: - bin_dir = path - current_entry = PathEntry.create( - bin_dir, is_root=True, only_python=True, pythons=pythons - ) - path_entries[bin_dir.as_posix()] = current_entry - return cls(base=bin_dir, paths=path_entries) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 527c5f0a..1595a963 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, print_function import logging +import operator from collections import defaultdict @@ -10,9 +11,15 @@ import sysconfig from vistir.compat import Path -from ..utils import ensure_path, optional_instance_of, get_python_version, filter_pythons -from .mixins import BaseFinder -from .path import VersionPath +from ..utils import ( + ensure_path, + optional_instance_of, + get_python_version, + filter_pythons, + unnest, +) +from .mixins import BaseFinder, BasePath +from .path import SystemPath, PathEntry from .python import PythonVersion @@ -20,51 +27,66 @@ logger = logging.getLogger(__name__) @attr.s -class PyenvFinder(BaseFinder): +class PyenvFinder(BaseFinder, BasePath): root = attr.ib(default=None, validator=optional_instance_of(Path)) - # ignore_unsupported should come before versions, because its value is used - # in versions's default initializer. - ignore_unsupported = attr.ib(default=False) + #: ignore_unsupported should come before versions, because its value is used + #: in versions's default initializer. + ignore_unsupported = attr.ib(default=True) + paths = attr.ib(default=attr.Factory(list)) + roots = attr.ib(default=attr.Factory(defaultdict)) versions = attr.ib() pythons = attr.ib() + @property + def expanded_paths(self): + return ( + path for path in unnest(p for p in self.versions.values()) + if path is not None + ) + @classmethod - def version_from_bin_dir(cls, base_dir): - pythons = [py for py in filter_pythons(base_dir)] + def version_from_bin_dir(cls, base_dir, name=None): py_version = None - for py in pythons: - version = get_python_version(py.as_posix()) - try: - py_version = PythonVersion.parse(version) - except Exception: - continue - if py_version: - return py_version - return + version_path = PathEntry.create( + path=base_dir.absolute().as_posix(), + only_python=True, + name=base_dir.parent.name, + ) + py_version = next(iter(version_path.find_all_python_versions()), None) + return py_version @versions.default def get_versions(self): - versions = defaultdict(VersionPath) + versions = defaultdict() bin_ = sysconfig._INSTALL_SCHEMES[sysconfig._get_default_scheme()]["scripts"] for p in self.root.glob("versions/*"): - if p.parent.name == "envs": + if p.parent.name == "envs" or p.name == "envs": continue + bin_dir = Path(bin_.format(base=p.as_posix())) + version_path = None + if bin_dir.exists(): + version_path = PathEntry.create( + path=bin_dir.absolute().as_posix(), + only_python=False, + name=p.name, + is_root=True, + ) + version = None try: version = PythonVersion.parse(p.name) except ValueError: - bin_dir = Path(bin_.format(base=p.as_posix())) - if bin_dir.exists(): - version = self.version_from_bin_dir(bin_dir) - if not version: - if not self.ignore_unsupported: - raise - continue + entry = next(iter(version_path.find_all_python_versions()), None) + if not entry: + if self.ignore_unsupported: + continue + raise + else: + version = entry.py_version.as_dict() except Exception: if not self.ignore_unsupported: raise logger.warning( - 'Unsupported Python version %r, ignoring...', - p.name, exc_info=True + "Unsupported Python version %r, ignoring...", p.name, exc_info=True ) continue if not version: @@ -75,24 +97,128 @@ class PyenvFinder(BaseFinder): version.get("patch"), version.get("is_prerelease"), version.get("is_devrelease"), - version.get("is_debug") - ) - versions[version_tuple] = VersionPath.create( - path=p.resolve(), only_python=True + version.get("is_debug"), ) + self.roots[p] = version_path + versions[version_tuple] = version_path + self.paths.append(version_path) return versions @pythons.default def get_pythons(self): pythons = defaultdict() - for v in self.versions.values(): - for p in v.paths.values(): - _path = ensure_path(p.path) - if p.is_python: - pythons[_path] = p + for p in self.paths: + pythons.update(p.pythons) return pythons @classmethod - def create(cls, root, ignore_unsupported=False): + def create(cls, root, ignore_unsupported=True): root = ensure_path(root) return cls(root=root, ignore_unsupported=ignore_unsupported) + + def find_all_python_versions( + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, + ): + """Search for a specific python version on the path. Return all copies + + :param major: Major python version to search for. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. + :rtype: List[:class:`~pythonfinder.models.PathEntry`] + """ + + version_matcher = operator.methodcaller( + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + ) + py = operator.attrgetter("as_python") + pythons = ( + py_ver for py_ver in (py(p) for p in self.pythons.values() if p is not None) + if py_ver is not None + ) + # pythons = filter(None, [p.as_python for p in self.pythons.values()]) + matching_versions = filter(lambda py: version_matcher(py), pythons) + version_sort = operator.attrgetter("version_sort") + return sorted(matching_versions, key=version_sort, reverse=True) + + def find_python_version( + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, + ): + """Search or self for the specified Python version and return the first match. + + :param major: Major version number. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. + """ + + version_matcher = operator.methodcaller( + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + ) + pythons = filter(None, [p.as_python for p in self.pythons.values()]) + matching_versions = filter(lambda py: version_matcher(py), pythons) + version_sort = operator.attrgetter("version_sort") + return next(iter(c for c in sorted(matching_versions, key=version_sort, reverse=True)), None) + + +@attr.s +class VersionPath(SystemPath): + base = attr.ib(default=None, validator=optional_instance_of(Path)) + name = attr.ib(default=None) + + @classmethod + def create(cls, path, only_python=True, pythons=None, name=None): + """Accepts a path to a base python version directory. + + Generates the pyenv version listings for it""" + path = ensure_path(path) + path_entries = defaultdict(PathEntry) + bin_ = sysconfig._INSTALL_SCHEMES[sysconfig._get_default_scheme()]["scripts"] + if path.as_posix().endswith(Path(bin_).name): + path = path.parent + bin_dir = ensure_path(bin_.format(base=path.as_posix())) + if not name: + name = path.name + current_entry = PathEntry.create( + bin_dir, is_root=True, only_python=True, pythons=pythons, name=name + ) + path_entries[bin_dir.as_posix()] = current_entry + return cls(name=name, base=bin_dir, paths=path_entries) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index c71b9d9b..ec99afe7 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -13,7 +13,11 @@ from packaging.version import parse as parse_version from ..environment import SYSTEM_ARCH from ..utils import ( - _filter_none, ensure_path, get_python_version, optional_instance_of + _filter_none, + ensure_path, + get_python_version, + optional_instance_of, + ensure_path, ) @@ -30,6 +34,7 @@ class PythonVersion(object): architecture = attr.ib(default=None) comes_from = attr.ib(default=None) executable = attr.ib(default=None) + name = attr.ib(default=None) @property def version_sort(self): @@ -65,22 +70,37 @@ class PythonVersion(object): self.patch, self.is_prerelease, self.is_devrelease, - self.is_debug + self.is_debug, ) def matches( - self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None, debug=False + self, + major=None, + minor=None, + patch=None, + pre=False, + dev=False, + arch=None, + debug=False, + name=None, ): - if arch and arch.isdigit(): - arch = "{0}bit".format(arch) + if arch: + own_arch = self.get_architecture() + if arch.isdigit(): + arch = "{0}bit".format(arch) return ( (major is None or self.major == major) and (minor is None or self.minor == minor) and (patch is None or self.patch == patch) and (pre is None or self.is_prerelease == pre) and (dev is None or self.is_devrelease == dev) - and (arch is None or self.architecture == arch) + and (arch is None or own_arch == arch) and (debug is None or self.is_debug == debug) + and ( + name is None + or (name and self.name) + and (self.name == name or self.name.startswith(name)) + ) ) def as_major(self): @@ -93,6 +113,18 @@ class PythonVersion(object): self_dict.update({"patch": None}) return self.create(**self_dict) + def as_dict(self): + return { + "major": self.major, + "minor": self.minor, + "patch": self.patch, + "is_prerelease": self.is_prerelease, + "is_postrelease": self.is_postrelease, + "is_devrelease": self.is_devrelease, + "is_debug": self.is_debug, + "version": self.version, + } + @classmethod def parse(cls, version): """Parse a valid version string into a dictionary @@ -138,8 +170,15 @@ class PythonVersion(object): "version": version, } + def get_architecture(self): + if self.architecture: + return self.architecture + arch, _ = platform.architecture(path.path.as_posix()) + self.architecture = arch + return self.architecture + @classmethod - def from_path(cls, path): + def from_path(cls, path, name=None): """Parses a python version from a system path. Raises: @@ -147,29 +186,33 @@ class PythonVersion(object): :param path: A string or :class:`~pythonfinder.models.path.PathEntry` :type path: str or :class:`~pythonfinder.models.path.PathEntry` instance - :param launcher_entry: A python launcher environment object. + :param str name: Name of the python distribution in question :return: An instance of a PythonVersion. :rtype: :class:`~pythonfinder.models.python.PythonVersion` """ from .path import PathEntry + from ..environment import IGNORE_UNSUPPORTED if not isinstance(path, PathEntry): - path = PathEntry.create(path, is_root=False, only_python=True) - if not path.is_python: + path = PathEntry.create(path, is_root=False, only_python=True, name=name) + if not path.is_python and not IGNORE_UNSUPPORTED: raise ValueError("Not a valid python path: %s" % path.path) return - py_version = get_python_version(str(path.path)) + py_version = get_python_version(path.path.as_posix()) instance_dict = cls.parse(py_version) - if not isinstance(instance_dict.get("version"), Version): + if not isinstance(instance_dict.get("version"), Version) and not IGNORE_UNSUPPORTED: raise ValueError("Not a valid python path: %s" % path.path) return - architecture, _ = platform.architecture(path.path.as_posix()) - instance_dict.update({"comes_from": path, "architecture": architecture}) + if not name: + name = path.name + instance_dict.update( + {"comes_from": path, "name": name} + ) return cls(**instance_dict) @classmethod - def from_windows_launcher(cls, launcher_entry): + def from_windows_launcher(cls, launcher_entry, name=None): """Create a new PythonVersion instance from a Windows Launcher Entry :param launcher_entry: A python launcher environment object. @@ -193,12 +236,14 @@ class PythonVersion(object): launcher_entry.info, "sys_architecture", SYSTEM_ARCH ), "executable": exe_path, + "name": name } ) py_version = cls.create(**creation_dict) - comes_from = PathEntry.create(exe_path, only_python=True) + comes_from = PathEntry.create(exe_path, only_python=True, name=name) comes_from.py_version = copy.deepcopy(py_version) py_version.comes_from = comes_from + py_version.name = comes_from.name return py_version @classmethod diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index fcb4d42a..e47bcc2c 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -22,7 +22,14 @@ class WindowsFinder(BaseFinder): pythons = attr.ib() def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): version_matcher = operator.methodcaller( "matches", @@ -32,6 +39,7 @@ class WindowsFinder(BaseFinder): pre=pre, dev=dev, arch=arch, + name=name, ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -40,13 +48,26 @@ class WindowsFinder(BaseFinder): return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): return next( ( v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=None, ) ), None, @@ -60,7 +81,7 @@ class WindowsFinder(BaseFinder): env_versions = pep514env.findall() path = None for version_object in env_versions: - install_path = getattr(version_object.info, 'install_path', None) + install_path = getattr(version_object.info, "install_path", None) if install_path is None: continue path = ensure_path(install_path.__getattr__("")) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index e965bb51..19a52e0a 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -7,13 +7,13 @@ from .models import SystemPath class Finder(object): - def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=False): + def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=True): """Finder A cross-platform Finder for locating python and other executables. Searches for python and other specified binaries starting in `path`, if supplied, but searching the bin path of `sys.executable` if `system=True`, and then searching in the `os.environ['PATH']` if `global_search=True`. When `global_search` - is `False`, this search operation is restricted to the allowed locations of + is `False`, this search operation is restricted to the allowed locations of `path` and `system`. :param path: A bin-directory search location, defaults to None @@ -57,7 +57,7 @@ class Finder(object): return self.system_path.which(exe) def find_python_version( - self, major, minor=None, patch=None, pre=None, dev=None, arch=None + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): from .models import PythonVersion @@ -69,12 +69,24 @@ class Finder(object): and patch is None ): if arch is None and "-" in major: - major, arch = major.rsplit("-", 1) - if not arch.isdigit(): - major = "{0}-{1}".format(major, arch) + orig_string = "{0!s}".format(major) + major, _, arch = major.rpartition("-") + if arch.startswith("x"): + arch = arch.lstrip("x") + if arch.lower().endswith("bit"): + arch = arch.lower().replace("bit", "") + if not (arch.isdigit() and (int(arch) & int(arch) - 1) == 0): + major = orig_string + arch = None else: arch = "{0}bit".format(arch) - version_dict = PythonVersion.parse(major) + try: + version_dict = PythonVersion.parse(major) + except ValueError: + if name is None: + name = "{0!s}".format(major) + major = None + version_dict = {} major = version_dict.get("major", major) minor = version_dict.get("minor", minor) patch = version_dict.get("patch", patch) @@ -83,16 +95,16 @@ class Finder(object): arch = version_dict.get("architecture", arch) if arch is None else arch if os.name == "nt": match = self.windows_finder.find_python_version( - major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) if match: return match return self.system_path.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): version_sort = operator.attrgetter("as_python.version_sort") python_version_dict = getattr(self.system_path, "python_version_dict") @@ -109,7 +121,7 @@ class Finder(object): paths = sorted(paths, key=version_sort, reverse=True) return paths versions = self.system_path.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) if not isinstance(versions, list): versions = [versions] diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index dced9eab..2f5a860d 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -16,10 +16,15 @@ import vistir from .exceptions import InvalidPythonVersion +try: + from functools import lru_cache +except ImportError: + from backports.functools_lru_cache import lru_cache + PYTHON_IMPLEMENTATIONS = ( "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", - "stackless", "activepython" + "stackless", "activepython", "micropython" ) RULES_BASE = ["*{0}", "*{0}?", "*{0}?.?", "*{0}?.?m"] RULES = [rule.format(impl) for impl in PYTHON_IMPLEMENTATIONS for rule in RULES_BASE] @@ -29,7 +34,17 @@ KNOWN_EXTS = KNOWN_EXTS | set( filter(None, os.environ.get("PATHEXT", "").split(os.pathsep)) ) +MATCH_RULES = [] +for rule in RULES: + MATCH_RULES.extend( + [ + "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) + for ext in KNOWN_EXTS + ] + ) + +@lru_cache(maxsize=128) def get_python_version(path): """Get python version string using subprocess from a given path.""" version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] @@ -54,6 +69,7 @@ def path_is_executable(path): return os.access(str(path), os.X_OK) +@lru_cache(maxsize=1024) def path_is_known_executable(path): return ( path_is_executable(path) @@ -62,24 +78,19 @@ def path_is_known_executable(path): ) +@lru_cache(maxsize=1024) def looks_like_python(name): - match_rules = [] - for rule in RULES: - match_rules.extend( - [ - "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) - for ext in KNOWN_EXTS - ] - ) if not any(name.lower().startswith(py_name) for py_name in PYTHON_IMPLEMENTATIONS): return False - return any(fnmatch(name, rule) for rule in match_rules) + return any(fnmatch(name, rule) for rule in MATCH_RULES) +@lru_cache(maxsize=128) def path_is_python(path): return path_is_executable(path) and looks_like_python(path.name) +@lru_cache(maxsize=1024) def ensure_path(path): """Given a path (either a string or a Path object), expand variables and return a Path object. @@ -90,13 +101,9 @@ def ensure_path(path): """ if isinstance(path, vistir.compat.Path): - path = path.as_posix() + return path path = vistir.compat.Path(os.path.expandvars(path)) - try: - path = path.resolve() - except OSError: - path = path.absolute() - return path + return path.absolute() def _filter_none(k, v): @@ -105,6 +112,7 @@ def _filter_none(k, v): return False +@lru_cache(maxsize=128) def filter_pythons(path): """Return all valid pythons in a given path""" if not isinstance(path, vistir.compat.Path): diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 1f3a2fcb..e0bc6746 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.1.9' +__version__ = '1.1.10' from .exceptions import RequirementError diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index f9ca97b8..bd76ca01 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -1,17 +1,19 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import json +import copy import os +import attr import plette.lockfiles import six -from vistir.compat import Path -from vistir.contextmanagers import atomic_open_for_write +from vistir.compat import Path, FileNotFoundError +from .project import ProjectFile from .requirements import Requirement +from .utils import optional_instance_of DEFAULT_NEWLINES = u"\n" @@ -22,47 +24,125 @@ def preferred_newlines(f): return DEFAULT_NEWLINES -class Lockfile(plette.lockfiles.Lockfile): - def __init__(self, *args, **kwargs): - path = kwargs.pop("path", None) - self._requirements = kwargs.pop("requirements", []) - self._dev_requirements = kwargs.pop("dev_requirements", []) - self.path = Path(path) if path else None - self.newlines = u"\n" - super(Lockfile, self).__init__(*args, **kwargs) +is_lockfile = optional_instance_of(plette.lockfiles.Lockfile) +is_projectfile = optional_instance_of(ProjectFile) + + +@attr.s(slots=True) +class Lockfile(object): + path = attr.ib(validator=optional_instance_of(Path), type=Path) + _requirements = attr.ib(default=attr.Factory(list), type=list) + _dev_requirements = attr.ib(default=attr.Factory(list), type=list) + projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) + _lockfile = attr.ib(validator=is_lockfile, type=plette.lockfiles.Lockfile) + newlines = attr.ib(default=DEFAULT_NEWLINES, type=six.text_type) + + @path.default + def _get_path(self): + return Path(os.curdir).absolute() + + @projectfile.default + def _get_projectfile(self): + return self.load_projectfile(self.path) + + @_lockfile.default + def _get_lockfile(self): + return self.projectfile.lockfile + + def __getattr__(self, k, *args, **kwargs): + retval = None + lockfile = super(Lockfile, self).__getattribute__("_lockfile") + try: + return super(Lockfile, self).__getattribute__(k) + except AttributeError: + retval = getattr(lockfile, k, None) + if not retval: + retval = super(Lockfile, self).__getattribute__(k, *args, **kwargs) + return retval @classmethod - def load(cls, path): + def read_projectfile(cls, path): + """Read the specified project file and provide an interface for writing/updating. + + :param str path: Path to the target file. + :return: A project file with the model and location for interaction + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + + pf = ProjectFile.read( + path, + plette.lockfiles.Lockfile, + invalid_ok=True + ) + return pf + + @classmethod + def load_projectfile(cls, path, create=True): + """Given a path, load or create the necessary lockfile. + + :param str path: Path to the project root or lockfile + :param bool create: Whether to create the lockfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the lockfile doesn't exist and ``create=False`` + :return: A project file instance for the supplied project + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + if not path: path = os.curdir path = Path(path).absolute() - if path.is_dir(): - path = path / "Pipfile.lock" - elif path.name == "Pipfile": - path = path.parent / "Pipfile.lock" - if not path.exists(): - raise OSError("Path does not exist: %s" % path) - return cls.create(path.parent, lockfile_name=path.name) + project_path = path if path.is_dir() else path.parent + lockfile_path = project_path / "Pipfile.lock" + if not project_path.exists(): + raise OSError("Project does not exist: %s" % project_path.as_posix()) + elif not lockfile_path.exists() and not create: + raise FileNotFoundError("Lockfile does not exist: %s" % lockfile_path.as_posix()) + projectfile = cls.read_projectfile(lockfile_path.as_posix()) + return projectfile @classmethod - def create(cls, project_path, lockfile_name="Pipfile.lock"): - """Create a new lockfile instance + def load(cls, path, create=True): + """Create a new lockfile instance. :param project_path: Path to project root - :type project_path: str or :class:`~pathlib.Path` - :returns: List[:class:`~requirementslib.Requirement`] objects + :type project_path: str or :class:`pathlib.Path` + :param str lockfile_name: Name of the lockfile in the project root directory + :param pipfile_path: Path to the project pipfile + :type pipfile_path: :class:`pathlib.Path` + :returns: A new lockfile representing the supplied project paths + :rtype: :class:`~requirementslib.models.lockfile.Lockfile` """ - if not isinstance(project_path, Path): - project_path = Path(project_path) - lockfile_path = project_path / lockfile_name - with lockfile_path.open(encoding="utf-8") as f: - lockfile = super(Lockfile, cls).load(f) - lockfile.newlines = preferred_newlines(f) - lockfile.path = lockfile_path - return lockfile + projectfile = cls.load_projectfile(path, create=create) + lockfile_path = Path(projectfile.location) + creation_args = { + "projectfile": projectfile, + "lockfile": projectfile.model, + "newlines": projectfile.line_ending, + "path": lockfile_path + } + return cls(**creation_args) + + @classmethod + def create(cls, path, create=True): + return cls.load(path, create=create) + + @property + def develop(self): + return self._lockfile.develop + + @property + def default(self): + return self._lockfile.default def get_requirements(self, dev=False): + """Produces a generator which generates requirements from the desired section. + + :param bool dev: Indicates whether to use dev requirements, defaults to False + :return: Requirements from the relevant the relevant pipfile + :rtype: :class:`~requirementslib.models.requirements.Requirement` + """ + section = self.develop if dev else self.default for k in section.keys(): yield Requirement.from_pipfile(k, section[k]._data) @@ -81,24 +161,26 @@ class Lockfile(plette.lockfiles.Lockfile): @property def dev_requirements_list(self): - return [{name: entry._data} for name, entry in self.develop.items()] + return [{name: entry._data} for name, entry in self._lockfile.develop.items()] @property def requirements_list(self): - return [{name: entry._data} for name, entry in self.develop.items()] + return [{name: entry._data} for name, entry in self._lockfile.default.items()] def write(self): - open_kwargs = {"newline": self.newlines} - with atomic_open_for_write(self.path.as_posix(), **open_kwargs) as f: - super(Lockfile, self).dump(f, encoding="utf-8") + self.projectfile.model = copy.deepcopy(self._lockfile) + self.projectfile.write() def as_requirements(self, include_hashes=False, dev=False): """Returns a list of requirements in pip-style format""" lines = [] section = self.dev_requirements if dev else self.requirements for req in section: - r = req.as_line() - if not include_hashes: - r = r.split("--hash", 1)[0] + kwargs = { + "include_hashes": include_hashes, + } + if req.editable: + kwargs["include_markers"] = False + r = req.as_line(**kwargs) lines.append(r.strip()) return lines diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 3a6f5b1e..94e9a2a1 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -1,64 +1,144 @@ # -*- coding: utf-8 -*- -from vistir.compat import Path + +from __future__ import absolute_import, unicode_literals, print_function + +import attr +import copy +import os + +from vistir.compat import Path, FileNotFoundError from .requirements import Requirement +from .project import ProjectFile +from .utils import optional_instance_of from ..exceptions import RequirementError import plette.pipfiles -class Pipfile(plette.pipfiles.Pipfile): +is_pipfile = optional_instance_of(plette.pipfiles.Pipfile) +is_path = optional_instance_of(Path) +is_projectfile = optional_instance_of(ProjectFile) + + +@attr.s(slots=True) +class Pipfile(object): + path = attr.ib(validator=is_path, type=Path) + projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) + _pipfile = attr.ib(type=plette.pipfiles.Pipfile) + requirements = attr.ib(default=attr.Factory(list), type=list) + dev_requirements = attr.ib(default=attr.Factory(list), type=list) + + @path.default + def _get_path(self): + return Path(os.curdir).absolute() + + @projectfile.default + def _get_projectfile(self): + return self.load_projectfile(os.curdir, create=False) + + @_pipfile.default + def _get_pipfile(self): + return self.projectfile.model + + def __getattr__(self, k, *args, **kwargs): + retval = None + pipfile = super(Pipfile, self).__getattribute__("_pipfile") + try: + return super(Pipfile, self).__getattribute__(k) + except AttributeError: + retval = getattr(pipfile, k, None) + if not retval: + retval = super(Pipfile, self).__getattribute__(k, *args, **kwargs) + return retval @property def requires_python(self): - return self.requires.requires_python + return self._pipfile.requires.requires_python @property def allow_prereleases(self): - return self.get("pipenv", {}).get("allow_prereleases", False) + return self._pipfile.get("pipenv", {}).get("allow_prereleases", False) @classmethod - def load(cls, path): + def read_projectfile(cls, path): + """Read the specified project file and provide an interface for writing/updating. + + :param str path: Path to the target file. + :return: A project file with the model and location for interaction + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + pf = ProjectFile.read( + path, + plette.pipfiles.Pipfile, + invalid_ok=True + ) + return pf + + @classmethod + def load_projectfile(cls, path, create=False): + """Given a path, load or create the necessary pipfile. + + :param str path: Path to the project root or pipfile + :param bool create: Whether to create the pipfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` + :return: A project file instance for the supplied project + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + if not path: + raise RuntimeError("Must pass a path to classmethod 'Pipfile.load'") if not isinstance(path, Path): path = Path(path) - pipfile_path = path / "Pipfile" - if not path.exists(): + pipfile_path = path if path.name == "Pipfile" else path.joinpath("Pipfile") + project_path = pipfile_path.parent + if not project_path.exists(): raise FileNotFoundError("%s is not a valid project path!" % path) elif not pipfile_path.exists() or not pipfile_path.is_file(): - raise RequirementError("%s is not a valid Pipfile" % pipfile_path) - with pipfile_path.open(encoding="utf-8") as fp: - pipfile = super(Pipfile, cls).load(fp) - pipfile.dev_requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.get("dev-packages", {}).items() - ] - pipfile.requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.get("packages", {}).items() - ] - pipfile.path = pipfile_path - return pipfile + if not create: + raise RequirementError("%s is not a valid Pipfile" % pipfile_path) + return cls.read_projectfile(pipfile_path.as_posix()) - # def resolve(self): - # It would be nice to still use this api someday - # option_sources = [s.expanded for s in self.sources] - # pip_args = [] - # if self.pipenv.allow_prereleases: - # pip_args.append('--pre') - # pip_options = get_pip_options(pip_args, sources=option_sources) - # finder = get_finder(sources=option_sources, pip_options=pip_options) - # resolver = DependencyResolver.create(finder=finder, allow_prereleases=self.pipenv.allow_prereleases) - # pkg_dict = {} - # for pkg in self.dev_packages.requirements + self.packages.requirements: - # pkg_dict[pkg.name] = pkg - # resolver.resolve(list(pkg_dict.values())) - # return resolver + @classmethod + def load(cls, path, create=False): + """Given a path, load or create the necessary pipfile. + + :param str path: Path to the project root or pipfile + :param bool create: Whether to create the pipfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` + :return: A pipfile instance pointing at the supplied project + :rtype:: class:`~requirementslib.models.pipfile.Pipfile` + """ + + projectfile = cls.load_projectfile(path, create=create) + pipfile = projectfile.model + dev_requirements = [ + Requirement.from_pipfile(k, v._data) for k, v in pipfile.get("dev-packages", {}).items() + ] + requirements = [ + Requirement.from_pipfile(k, v._data) for k, v in pipfile.get("packages", {}).items() + ] + creation_args = { + "projectfile": projectfile, + "pipfile": pipfile, + "dev_requirements": dev_requirements, + "requirements": requirements, + "path": Path(projectfile.location) + } + return cls(**creation_args) + + def write(self): + self.projectfile.model = copy.deepcopy(self._pipfile) + self.projectfile.write() @property def dev_packages(self, as_requirements=True): if as_requirements: return self.dev_requirements - return self.get('dev-packages', {}) + return self._pipfile.get('dev-packages', {}) @property def packages(self, as_requirements=True): if as_requirements: return self.requirements - return self.get('packages', {}) + return self._pipfile.get('packages', {}) diff --git a/pipenv/vendor/requirementslib/models/project.py b/pipenv/vendor/requirementslib/models/project.py new file mode 100644 index 00000000..f6e037d6 --- /dev/null +++ b/pipenv/vendor/requirementslib/models/project.py @@ -0,0 +1,241 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, unicode_literals + +import collections +import io +import os + +import attr +import packaging.markers +import packaging.utils +import plette +import plette.models +import six +import tomlkit + + +SectionDifference = collections.namedtuple("SectionDifference", [ + "inthis", "inthat", +]) +FileDifference = collections.namedtuple("FileDifference", [ + "default", "develop", +]) + + +def _are_pipfile_entries_equal(a, b): + a = {k: v for k, v in a.items() if k not in ("markers", "hashes", "hash")} + b = {k: v for k, v in b.items() if k not in ("markers", "hashes", "hash")} + if a != b: + return False + try: + marker_eval_a = packaging.markers.Marker(a["markers"]).evaluate() + except (AttributeError, KeyError, TypeError, ValueError): + marker_eval_a = True + try: + marker_eval_b = packaging.markers.Marker(b["markers"]).evaluate() + except (AttributeError, KeyError, TypeError, ValueError): + marker_eval_b = True + return marker_eval_a == marker_eval_b + + +DEFAULT_NEWLINES = "\n" + + +def preferred_newlines(f): + if isinstance(f.newlines, six.text_type): + return f.newlines + return DEFAULT_NEWLINES + + +@attr.s +class ProjectFile(object): + """A file in the Pipfile project. + """ + location = attr.ib() + line_ending = attr.ib() + model = attr.ib() + + @classmethod + def read(cls, location, model_cls, invalid_ok=False): + try: + with io.open(location, encoding="utf-8") as f: + model = model_cls.load(f) + line_ending = preferred_newlines(f) + except Exception: + if not invalid_ok: + raise + model = None + line_ending = DEFAULT_NEWLINES + return cls(location=location, line_ending=line_ending, model=model) + + def write(self): + kwargs = {"encoding": "utf-8", "newline": self.line_ending} + with io.open(self.location, "w", **kwargs) as f: + self.model.dump(f) + + def dumps(self): + strio = six.StringIO() + self.model.dump(strio) + return strio.getvalue() + + +@attr.s +class Project(object): + + root = attr.ib() + _p = attr.ib(init=False) + _l = attr.ib(init=False) + + def __attrs_post_init__(self): + self.root = root = os.path.abspath(self.root) + self._p = ProjectFile.read( + os.path.join(root, "Pipfile"), + plette.Pipfile, + ) + self._l = ProjectFile.read( + os.path.join(root, "Pipfile.lock"), + plette.Lockfile, + invalid_ok=True, + ) + + @property + def pipfile(self): + return self._p.model + + @property + def pipfile_location(self): + return self._p.location + + @property + def lockfile(self): + return self._l.model + + @property + def lockfile_location(self): + return self._l.location + + @lockfile.setter + def lockfile(self, new): + self._l.model = new + + def is_synced(self): + return self.lockfile and self.lockfile.is_up_to_date(self.pipfile) + + def _get_pipfile_section(self, develop, insert=True): + name = "dev-packages" if develop else "packages" + try: + section = self.pipfile[name] + except KeyError: + section = plette.models.PackageCollection(tomlkit.table()) + if insert: + self.pipfile[name] = section + return section + + def contains_key_in_pipfile(self, key): + sections = [ + self._get_pipfile_section(develop=False, insert=False), + self._get_pipfile_section(develop=True, insert=False), + ] + return any( + (packaging.utils.canonicalize_name(name) == + packaging.utils.canonicalize_name(key)) + for section in sections + for name in section + ) + + def add_line_to_pipfile(self, line, develop): + from requirementslib import Requirement + requirement = Requirement.from_line(line) + section = self._get_pipfile_section(develop=develop) + key = requirement.normalized_name + entry = next(iter(requirement.as_pipfile().values())) + if isinstance(entry, dict): + # HACK: TOMLKit prefers to expand tables by default, but we + # always want inline tables here. Also tomlkit.inline_table + # does not have `update()`. + table = tomlkit.inline_table() + for k, v in entry.items(): + table[k] = v + entry = table + section[key] = entry + + def remove_keys_from_pipfile(self, keys, default, develop): + keys = {packaging.utils.canonicalize_name(key) for key in keys} + sections = [] + if default: + sections.append(self._get_pipfile_section( + develop=False, insert=False, + )) + if develop: + sections.append(self._get_pipfile_section( + develop=True, insert=False, + )) + for section in sections: + removals = set() + for name in section: + if packaging.utils.canonicalize_name(name) in keys: + removals.add(name) + for key in removals: + del section._data[key] + + def remove_keys_from_lockfile(self, keys): + keys = {packaging.utils.canonicalize_name(key) for key in keys} + removed = False + for section_name in ("default", "develop"): + try: + section = self.lockfile[section_name] + except KeyError: + continue + removals = set() + for name in section: + if packaging.utils.canonicalize_name(name) in keys: + removals.add(name) + removed = removed or bool(removals) + for key in removals: + del section._data[key] + + if removed: + # HACK: The lock file no longer represents the Pipfile at this + # point. Set the hash to an arbitrary invalid value. + self.lockfile.meta.hash = plette.models.Hash({"__invalid__": ""}) + + def difference_lockfile(self, lockfile): + """Generate a difference between the current and given lockfiles. + + Returns a 2-tuple containing differences in default in develop + sections. + + Each element is a 2-tuple of dicts. The first, `inthis`, contains + entries only present in the current lockfile; the second, `inthat`, + contains entries only present in the given one. + + If a key exists in both this and that, but the values differ, the key + is present in both dicts, pointing to values from each file. + """ + diff_data = { + "default": SectionDifference({}, {}), + "develop": SectionDifference({}, {}), + } + for section_name, section_diff in diff_data.items(): + try: + this = self.lockfile[section_name]._data + except (KeyError, TypeError): + this = {} + try: + that = lockfile[section_name]._data + except (KeyError, TypeError): + that = {} + for key, this_value in this.items(): + try: + that_value = that[key] + except KeyError: + section_diff.inthis[key] = this_value + continue + if not _are_pipfile_entries_equal(this_value, that_value): + section_diff.inthis[key] = this_value + section_diff.inthat[key] = that_value + for key, that_value in that.items(): + if key not in this: + section_diff.inthat[key] = that_value + return FileDifference(**diff_data) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 248ca777..c2768417 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- + from __future__ import absolute_import -import atexit import collections import hashlib import os @@ -9,6 +9,7 @@ import os from contextlib import contextmanager import attr +import six from first import first from packaging.markers import Marker @@ -42,7 +43,8 @@ from .utils import ( is_pinned_requirement, make_install_requirement, optional_instance_of, parse_extras, specs_to_string, split_markers_from_line, split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs + validate_specifiers, validate_vcs, normalize_name, + Requirement as PkgResourcesRequirement ) from .vcs import VCSRepository @@ -99,7 +101,7 @@ class NamedRequirement(BaseRequirement): # FIXME: This should actually be canonicalized but for now we have to # simply lowercase it and replace underscores, since full canonicalization # also replaces dots and that doesn't actually work when querying the index - return "{0}".format(self.name.lower().replace("_", "-")) + return "{0}".format(normalize_name(self.name)) @property def pipfile_part(self): @@ -123,12 +125,12 @@ class FileRequirement(BaseRequirement): setup_path = attr.ib(default=None) path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) # : path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - editable = attr.ib(default=None) - extras = attr.ib(default=attr.Factory(list)) - uri = attr.ib() - link = attr.ib() - name = attr.ib() - req = attr.ib() + editable = attr.ib(default=False, type=bool) + extras = attr.ib(default=attr.Factory(list), type=list) + uri = attr.ib(type=six.string_types) + link = attr.ib(type=Link) + name = attr.ib(type=six.string_types) + req = attr.ib(type=PkgResourcesRequirement) _has_hashed_name = False _uri_scheme = attr.ib(default=None) @@ -297,7 +299,7 @@ class FileRequirement(BaseRequirement): @req.default def get_requirement(self): - req = init_requirement(canonicalize_name(self.name)) + req = init_requirement(normalize_name(self.name)) req.editable = False req.line = self.link.url_without_fragment if self.path and self.link and self.link.scheme.startswith("file"): @@ -948,7 +950,8 @@ class Requirement(object): cls_inst.req.req.line = cls_inst.as_line() return cls_inst - def as_line(self, sources=None, include_hashes=True, include_extras=True, as_list=False): + def as_line(self, sources=None, include_hashes=True, include_extras=True, + include_markers=True, as_list=False): """Format this requirement as a line in requirements.txt. If ``sources`` provided, it should be an sequence of mappings, containing @@ -967,7 +970,7 @@ class Requirement(object): self.req.line_part, self.extras_as_pip if include_extras else "", self.specifiers if include_specifiers else "", - self.markers_as_pip, + self.markers_as_pip if include_markers else "", ] if as_list: # This is used for passing to a subprocess call diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index cba63295..7350d0ac 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -508,3 +508,15 @@ def fix_requires_python_marker(requires_python): ]) marker_to_add = PackagingRequirement('fakepkg; {0}'.format(marker_str)).marker return marker_to_add + + +def normalize_name(pkg): + """Given a package name, return its normalized, non-canonicalized form. + + :param str pkg: The name of a package + :return: A normalized package name + :rtype: str + """ + + assert isinstance(pkg, six.string_types) + return pkg.replace("_", "-").lower() diff --git a/pipenv/vendor/shellingham/__init__.py b/pipenv/vendor/shellingham/__init__.py index 90c00abb..576c4224 100644 --- a/pipenv/vendor/shellingham/__init__.py +++ b/pipenv/vendor/shellingham/__init__.py @@ -4,7 +4,7 @@ import os from ._core import ShellDetectionFailure -__version__ = '1.2.6' +__version__ = '1.2.7' def detect_shell(pid=None, max_depth=6): diff --git a/pipenv/vendor/shellingham/posix/ps.py b/pipenv/vendor/shellingham/posix/ps.py index ab4c2a9e..4a155ed5 100644 --- a/pipenv/vendor/shellingham/posix/ps.py +++ b/pipenv/vendor/shellingham/posix/ps.py @@ -21,6 +21,12 @@ def get_process_mapping(): if e.errno != errno.ENOENT: raise raise PsNotAvailable('ps not found') + except subprocess.CalledProcessError as e: + # `ps` can return 1 if the process list is completely empty. + # (sarugaku/shellingham#15) + if not e.output.strip(): + return {} + raise if not isinstance(output, str): encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() output = output.decode(encoding) @@ -28,9 +34,9 @@ def get_process_mapping(): for line in output.split('\n'): try: pid, ppid, args = line.strip().split(None, 2) + processes[pid] = Process( + args=tuple(shlex.split(args)), pid=pid, ppid=ppid, + ) except ValueError: continue - processes[pid] = Process( - args=tuple(shlex.split(args)), pid=pid, ppid=ppid, - ) return processes diff --git a/pipenv/vendor/toml.LICENSE b/pipenv/vendor/toml.LICENSE deleted file mode 100644 index d8b406c9..00000000 --- a/pipenv/vendor/toml.LICENSE +++ /dev/null @@ -1,26 +0,0 @@ -The MIT License - -Copyright 2013-2017 Uiri Noyb -Copyright 2015-2016 Julien Enselme -Copyright 2016 Google Inc. -Copyright 2017 Samuel Vasko -Copyright 2017 Nate Prewitt -Copyright 2017 Jack Evans - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file diff --git a/pipenv/vendor/toml.py b/pipenv/vendor/toml.py deleted file mode 100644 index dac39883..00000000 --- a/pipenv/vendor/toml.py +++ /dev/null @@ -1,1039 +0,0 @@ -"""Python module which parses and emits TOML. - -Released under the MIT license. -""" -import re -import io -import datetime -from os import linesep -import sys - -__version__ = "0.9.6" -_spec_ = "0.4.0" - - -class TomlDecodeError(Exception): - """Base toml Exception / Error.""" - pass - - -class TomlTz(datetime.tzinfo): - def __init__(self, toml_offset): - if toml_offset == "Z": - self._raw_offset = "+00:00" - else: - self._raw_offset = toml_offset - self._sign = -1 if self._raw_offset[0] == '-' else 1 - self._hours = int(self._raw_offset[1:3]) - self._minutes = int(self._raw_offset[4:6]) - - def tzname(self, dt): - return "UTC" + self._raw_offset - - def utcoffset(self, dt): - return self._sign * datetime.timedelta(hours=self._hours, - minutes=self._minutes) - - def dst(self, dt): - return datetime.timedelta(0) - - -class InlineTableDict(object): - """Sentinel subclass of dict for inline tables.""" - - -def _get_empty_inline_table(_dict): - class DynamicInlineTableDict(_dict, InlineTableDict): - """Concrete sentinel subclass for inline tables. - It is a subclass of _dict which is passed in dynamically at load time - It is also a subclass of InlineTableDict - """ - - return DynamicInlineTableDict() - - -try: - _range = xrange -except NameError: - unicode = str - _range = range - basestring = str - unichr = chr - -try: - FNFError = FileNotFoundError -except NameError: - FNFError = IOError - - -def load(f, _dict=dict): - """Parses named file or files as toml and returns a dictionary - - Args: - f: Path to the file to open, array of files to read into single dict - or a file descriptor - _dict: (optional) Specifies the class of the returned toml dictionary - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError -- When f is invalid type - TomlDecodeError: Error while decoding toml - IOError / FileNotFoundError -- When an array with no valid (existing) - (Python 2 / Python 3) file paths is passed - """ - - if isinstance(f, basestring): - with io.open(f, encoding='utf-8') as ffile: - return loads(ffile.read(), _dict) - elif isinstance(f, list): - from os import path as op - from warnings import warn - if not [path for path in f if op.exists(path)]: - error_msg = "Load expects a list to contain filenames only." - error_msg += linesep - error_msg += ("The list needs to contain the path of at least one " - "existing file.") - raise FNFError(error_msg) - d = _dict() - for l in f: - if op.exists(l): - d.update(load(l)) - else: - warn("Non-existent filename in list with at least one valid " - "filename") - return d - else: - try: - return loads(f.read(), _dict) - except AttributeError: - raise TypeError("You can only load a file descriptor, filename or " - "list") - - -_groupname_re = re.compile(r'^[A-Za-z0-9_-]+$') - - -def loads(s, _dict=dict): - """Parses string as toml - - Args: - s: String to be parsed - _dict: (optional) Specifies the class of the returned toml dictionary - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError: When a non-string is passed - TomlDecodeError: Error while decoding toml - """ - - implicitgroups = [] - retval = _dict() - currentlevel = retval - if not isinstance(s, basestring): - raise TypeError("Expecting something like a string") - - if not isinstance(s, unicode): - s = s.decode('utf8') - - sl = list(s) - openarr = 0 - openstring = False - openstrchar = "" - multilinestr = False - arrayoftables = False - beginline = True - keygroup = False - keyname = 0 - for i, item in enumerate(sl): - if item == '\r' and sl[i + 1] == '\n': - sl[i] = ' ' - continue - if keyname: - if item == '\n': - raise TomlDecodeError("Key name found without value." - " Reached end of line.") - if openstring: - if item == openstrchar: - keyname = 2 - openstring = False - openstrchar = "" - continue - elif keyname == 1: - if item.isspace(): - keyname = 2 - continue - elif item.isalnum() or item == '_' or item == '-': - continue - elif keyname == 2 and item.isspace(): - continue - if item == '=': - keyname = 0 - else: - raise TomlDecodeError("Found invalid character in key name: '" + - item + "'. Try quoting the key name.") - if item == "'" and openstrchar != '"': - k = 1 - try: - while sl[i - k] == "'": - k += 1 - if k == 3: - break - except IndexError: - pass - if k == 3: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = "'" - else: - openstrchar = "" - if item == '"' and openstrchar != "'": - oddbackslash = False - k = 1 - tripquote = False - try: - while sl[i - k] == '"': - k += 1 - if k == 3: - tripquote = True - break - if k == 1 or (k == 3 and tripquote): - while sl[i - k] == '\\': - oddbackslash = not oddbackslash - k += 1 - except IndexError: - pass - if not oddbackslash: - if tripquote: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = '"' - else: - openstrchar = "" - if item == '#' and (not openstring and not keygroup and - not arrayoftables): - j = i - try: - while sl[j] != '\n': - sl[j] = ' ' - j += 1 - except IndexError: - break - if item == '[' and (not openstring and not keygroup and - not arrayoftables): - if beginline: - if len(sl) > i + 1 and sl[i + 1] == '[': - arrayoftables = True - else: - keygroup = True - else: - openarr += 1 - if item == ']' and not openstring: - if keygroup: - keygroup = False - elif arrayoftables: - if sl[i - 1] == ']': - arrayoftables = False - else: - openarr -= 1 - if item == '\n': - if openstring or multilinestr: - if not multilinestr: - raise TomlDecodeError("Unbalanced quotes") - if ((sl[i - 1] == "'" or sl[i - 1] == '"') and ( - sl[i - 2] == sl[i - 1])): - sl[i] = sl[i - 1] - if sl[i - 3] == sl[i - 1]: - sl[i - 3] = ' ' - elif openarr: - sl[i] = ' ' - else: - beginline = True - elif beginline and sl[i] != ' ' and sl[i] != '\t': - beginline = False - if not keygroup and not arrayoftables: - if sl[i] == '=': - raise TomlDecodeError("Found empty keyname. ") - keyname = 1 - s = ''.join(sl) - s = s.split('\n') - multikey = None - multilinestr = "" - multibackslash = False - for line in s: - if not multilinestr or multibackslash or '\n' not in multilinestr: - line = line.strip() - if line == "" and (not multikey or multibackslash): - continue - if multikey: - if multibackslash: - multilinestr += line - else: - multilinestr += line - multibackslash = False - if len(line) > 2 and (line[-1] == multilinestr[0] and - line[-2] == multilinestr[0] and - line[-3] == multilinestr[0]): - try: - value, vtype = _load_value(multilinestr, _dict) - except ValueError as err: - raise TomlDecodeError(str(err)) - currentlevel[multikey] = value - multikey = None - multilinestr = "" - else: - k = len(multilinestr) - 1 - while k > -1 and multilinestr[k] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = multilinestr[:-1] - else: - multilinestr += "\n" - continue - if line[0] == '[': - arrayoftables = False - if len(line) == 1: - raise TomlDecodeError("Opening key group bracket on line by " - "itself.") - if line[1] == '[': - arrayoftables = True - line = line[2:] - splitstr = ']]' - else: - line = line[1:] - splitstr = ']' - i = 1 - quotesplits = _get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and splitstr in quotesplit: - break - i += quotesplit.count(splitstr) - quoted = not quoted - line = line.split(splitstr, i) - if len(line) < i + 1 or line[-1].strip() != "": - raise TomlDecodeError("Key group not on a line by itself.") - groups = splitstr.join(line[:-1]).split('.') - i = 0 - while i < len(groups): - groups[i] = groups[i].strip() - if len(groups[i]) > 0 and (groups[i][0] == '"' or - groups[i][0] == "'"): - groupstr = groups[i] - j = i + 1 - while not groupstr[0] == groupstr[-1]: - j += 1 - if j > len(groups) + 2: - raise TomlDecodeError("Invalid group name '" + - groupstr + "' Something " + - "went wrong.") - groupstr = '.'.join(groups[i:j]).strip() - groups[i] = groupstr[1:-1] - groups[i + 1:j] = [] - else: - if not _groupname_re.match(groups[i]): - raise TomlDecodeError("Invalid group name '" + - groups[i] + "'. Try quoting it.") - i += 1 - currentlevel = retval - for i in _range(len(groups)): - group = groups[i] - if group == "": - raise TomlDecodeError("Can't have a keygroup with an empty " - "name") - try: - currentlevel[group] - if i == len(groups) - 1: - if group in implicitgroups: - implicitgroups.remove(group) - if arrayoftables: - raise TomlDecodeError("An implicitly defined " - "table can't be an array") - elif arrayoftables: - currentlevel[group].append(_dict()) - else: - raise TomlDecodeError("What? " + group + - " already exists?" + - str(currentlevel)) - except TypeError: - currentlevel = currentlevel[-1] - try: - currentlevel[group] - except KeyError: - currentlevel[group] = _dict() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [_dict()] - except KeyError: - if i != len(groups) - 1: - implicitgroups.append(group) - currentlevel[group] = _dict() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [_dict()] - currentlevel = currentlevel[group] - if arrayoftables: - try: - currentlevel = currentlevel[-1] - except KeyError: - pass - elif line[0] == "{": - if line[-1] != "}": - raise TomlDecodeError("Line breaks are not allowed in inline" - "objects") - try: - _load_inline_object(line, currentlevel, _dict, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err)) - elif "=" in line: - try: - ret = _load_line(line, currentlevel, _dict, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err)) - if ret is not None: - multikey, multilinestr, multibackslash = ret - return retval - - -def _load_inline_object(line, currentlevel, _dict, multikey=False, - multibackslash=False): - candidate_groups = line[1:-1].split(",") - groups = [] - if len(candidate_groups) == 1 and not candidate_groups[0].strip(): - candidate_groups.pop() - while len(candidate_groups) > 0: - candidate_group = candidate_groups.pop(0) - try: - _, value = candidate_group.split('=', 1) - except ValueError: - raise ValueError("Invalid inline table encountered") - value = value.strip() - if ((value[0] == value[-1] and value[0] in ('"', "'")) or ( - value[0] in '-0123456789' or - value in ('true', 'false') or - (value[0] == "[" and value[-1] == "]") or - (value[0] == '{' and value[-1] == '}'))): - groups.append(candidate_group) - elif len(candidate_groups) > 0: - candidate_groups[0] = candidate_group + "," + candidate_groups[0] - else: - raise ValueError("Invalid inline table value encountered") - for group in groups: - status = _load_line(group, currentlevel, _dict, multikey, - multibackslash) - if status is not None: - break - - -# Matches a TOML number, which allows underscores for readability -_number_with_underscores = re.compile('([0-9])(_([0-9]))*') - - -def _strictly_valid_num(n): - n = n.strip() - if not n: - return False - if n[0] == '_': - return False - if n[-1] == '_': - return False - if "_." in n or "._" in n: - return False - if len(n) == 1: - return True - if n[0] == '0' and n[1] != '.': - return False - if n[0] == '+' or n[0] == '-': - n = n[1:] - if n[0] == '0' and n[1] != '.': - return False - if '__' in n: - return False - return True - - -def _get_split_on_quotes(line): - doublequotesplits = line.split('"') - quoted = False - quotesplits = [] - if len(doublequotesplits) > 1 and "'" in doublequotesplits[0]: - singlequotesplits = doublequotesplits[0].split("'") - doublequotesplits = doublequotesplits[1:] - while len(singlequotesplits) % 2 == 0 and len(doublequotesplits): - singlequotesplits[-1] += '"' + doublequotesplits[0] - doublequotesplits = doublequotesplits[1:] - if "'" in singlequotesplits[-1]: - singlequotesplits = (singlequotesplits[:-1] + - singlequotesplits[-1].split("'")) - quotesplits += singlequotesplits - for doublequotesplit in doublequotesplits: - if quoted: - quotesplits.append(doublequotesplit) - else: - quotesplits += doublequotesplit.split("'") - quoted = not quoted - return quotesplits - - -def _load_line(line, currentlevel, _dict, multikey, multibackslash): - i = 1 - quotesplits = _get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and '=' in quotesplit: - break - i += quotesplit.count('=') - quoted = not quoted - pair = line.split('=', i) - strictly_valid = _strictly_valid_num(pair[-1]) - if _number_with_underscores.match(pair[-1]): - pair[-1] = pair[-1].replace('_', '') - while len(pair[-1]) and (pair[-1][0] != ' ' and pair[-1][0] != '\t' and - pair[-1][0] != "'" and pair[-1][0] != '"' and - pair[-1][0] != '[' and pair[-1][0] != '{' and - pair[-1] != 'true' and pair[-1] != 'false'): - try: - float(pair[-1]) - break - except ValueError: - pass - if _load_date(pair[-1]) is not None: - break - i += 1 - prev_val = pair[-1] - pair = line.split('=', i) - if prev_val == pair[-1]: - raise ValueError("Invalid date or number") - if strictly_valid: - strictly_valid = _strictly_valid_num(pair[-1]) - pair = ['='.join(pair[:-1]).strip(), pair[-1].strip()] - if (pair[0][0] == '"' or pair[0][0] == "'") and \ - (pair[0][-1] == '"' or pair[0][-1] == "'"): - pair[0] = pair[0][1:-1] - if len(pair[1]) > 2 and ((pair[1][0] == '"' or pair[1][0] == "'") and - pair[1][1] == pair[1][0] and - pair[1][2] == pair[1][0] and - not (len(pair[1]) > 5 and - pair[1][-1] == pair[1][0] and - pair[1][-2] == pair[1][0] and - pair[1][-3] == pair[1][0])): - k = len(pair[1]) - 1 - while k > -1 and pair[1][k] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = pair[1][:-1] - else: - multilinestr = pair[1] + "\n" - multikey = pair[0] - else: - value, vtype = _load_value(pair[1], _dict, strictly_valid) - try: - currentlevel[pair[0]] - raise ValueError("Duplicate keys!") - except KeyError: - if multikey: - return multikey, multilinestr, multibackslash - else: - currentlevel[pair[0]] = value - - -def _load_date(val): - microsecond = 0 - tz = None - try: - if len(val) > 19: - if val[19] == '.': - if val[-1].upper() == 'Z': - subsecondval = val[20:-1] - tzval = "Z" - else: - subsecondvalandtz = val[20:] - if '+' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('+') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - elif '-' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('-') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - tz = TomlTz(tzval) - microsecond = int(int(subsecondval) * - (10 ** (6 - len(subsecondval)))) - else: - tz = TomlTz(val[19:]) - except ValueError: - tz = None - if "-" not in val[1:]: - return None - try: - d = datetime.datetime( - int(val[:4]), int(val[5:7]), - int(val[8:10]), int(val[11:13]), - int(val[14:16]), int(val[17:19]), microsecond, tz) - except ValueError: - return None - return d - - -def _load_unicode_escapes(v, hexbytes, prefix): - skip = False - i = len(v) - 1 - while i > -1 and v[i] == '\\': - skip = not skip - i -= 1 - for hx in hexbytes: - if skip: - skip = False - i = len(hx) - 1 - while i > -1 and hx[i] == '\\': - skip = not skip - i -= 1 - v += prefix - v += hx - continue - hxb = "" - i = 0 - hxblen = 4 - if prefix == "\\U": - hxblen = 8 - hxb = ''.join(hx[i:i + hxblen]).lower() - if hxb.strip('0123456789abcdef'): - raise ValueError("Invalid escape sequence: " + hxb) - if hxb[0] == "d" and hxb[1].strip('01234567'): - raise ValueError("Invalid escape sequence: " + hxb + - ". Only scalar unicode points are allowed.") - v += unichr(int(hxb, 16)) - v += unicode(hx[len(hxb):]) - return v - - -# Unescape TOML string values. - -# content after the \ -_escapes = ['0', 'b', 'f', 'n', 'r', 't', '"'] -# What it should be replaced by -_escapedchars = ['\0', '\b', '\f', '\n', '\r', '\t', '\"'] -# Used for substitution -_escape_to_escapedchars = dict(zip(_escapes, _escapedchars)) - - -def _unescape(v): - """Unescape characters in a TOML string.""" - i = 0 - backslash = False - while i < len(v): - if backslash: - backslash = False - if v[i] in _escapes: - v = v[:i - 1] + _escape_to_escapedchars[v[i]] + v[i + 1:] - elif v[i] == '\\': - v = v[:i - 1] + v[i:] - elif v[i] == 'u' or v[i] == 'U': - i += 1 - else: - raise ValueError("Reserved escape sequence used") - continue - elif v[i] == '\\': - backslash = True - i += 1 - return v - - -def _load_value(v, _dict, strictly_valid=True): - if not v: - raise ValueError("Empty value is invalid") - if v == 'true': - return (True, "bool") - elif v == 'false': - return (False, "bool") - elif v[0] == '"': - testv = v[1:].split('"') - triplequote = False - triplequotecount = 0 - if len(testv) > 1 and testv[0] == '' and testv[1] == '': - testv = testv[2:] - triplequote = True - closed = False - for tv in testv: - if tv == '': - if triplequote: - triplequotecount += 1 - else: - closed = True - else: - oddbackslash = False - try: - i = -1 - j = tv[i] - while j == '\\': - oddbackslash = not oddbackslash - i -= 1 - j = tv[i] - except IndexError: - pass - if not oddbackslash: - if closed: - raise ValueError("Stuff after closed string. WTF?") - else: - if not triplequote or triplequotecount > 1: - closed = True - else: - triplequotecount = 0 - escapeseqs = v.split('\\')[1:] - backslash = False - for i in escapeseqs: - if i == '': - backslash = not backslash - else: - if i[0] not in _escapes and (i[0] != 'u' and i[0] != 'U' and - not backslash): - raise ValueError("Reserved escape sequence used") - if backslash: - backslash = False - for prefix in ["\\u", "\\U"]: - if prefix in v: - hexbytes = v.split(prefix) - v = _load_unicode_escapes(hexbytes[0], hexbytes[1:], prefix) - v = _unescape(v) - if len(v) > 1 and v[1] == '"' and (len(v) < 3 or v[1] == v[2]): - v = v[2:-2] - return (v[1:-1], "str") - elif v[0] == "'": - if v[1] == "'" and (len(v) < 3 or v[1] == v[2]): - v = v[2:-2] - return (v[1:-1], "str") - elif v[0] == '[': - return (_load_array(v, _dict), "array") - elif v[0] == '{': - inline_object = _get_empty_inline_table(_dict) - _load_inline_object(v, inline_object, _dict) - return (inline_object, "inline_object") - else: - parsed_date = _load_date(v) - if parsed_date is not None: - return (parsed_date, "date") - if not strictly_valid: - raise ValueError("Weirdness with leading zeroes or " - "underscores in your number.") - itype = "int" - neg = False - if v[0] == '-': - neg = True - v = v[1:] - elif v[0] == '+': - v = v[1:] - v = v.replace('_', '') - if '.' in v or 'e' in v or 'E' in v: - if '.' in v and v.split('.', 1)[1] == '': - raise ValueError("This float is missing digits after " - "the point") - if v[0] not in '0123456789': - raise ValueError("This float doesn't have a leading digit") - v = float(v) - itype = "float" - else: - v = int(v) - if neg: - return (0 - v, itype) - return (v, itype) - - -def _bounded_string(s): - if len(s) == 0: - return True - if s[-1] != s[0]: - return False - i = -2 - backslash = False - while len(s) + i > 0: - if s[i] == "\\": - backslash = not backslash - i -= 1 - else: - break - return not backslash - - -def _load_array(a, _dict): - atype = None - retval = [] - a = a.strip() - if '[' not in a[1:-1] or "" != a[1:-1].split('[')[0].strip(): - strarray = False - tmpa = a[1:-1].strip() - if tmpa != '' and (tmpa[0] == '"' or tmpa[0] == "'"): - strarray = True - if not a[1:-1].strip().startswith('{'): - a = a[1:-1].split(',') - else: - # a is an inline object, we must find the matching parenthesis - # to define groups - new_a = [] - start_group_index = 1 - end_group_index = 2 - in_str = False - while end_group_index < len(a[1:]): - if a[end_group_index] == '"' or a[end_group_index] == "'": - if in_str: - backslash_index = end_group_index - 1 - while (backslash_index > -1 and - a[backslash_index] == '\\'): - in_str = not in_str - backslash_index -= 1 - in_str = not in_str - if in_str or a[end_group_index] != '}': - end_group_index += 1 - continue - - # Increase end_group_index by 1 to get the closing bracket - end_group_index += 1 - new_a.append(a[start_group_index:end_group_index]) - - # The next start index is at least after the closing bracket, a - # closing bracket can be followed by a comma since we are in - # an array. - start_group_index = end_group_index + 1 - while (start_group_index < len(a[1:]) and - a[start_group_index] != '{'): - start_group_index += 1 - end_group_index = start_group_index + 1 - a = new_a - b = 0 - if strarray: - while b < len(a) - 1: - ab = a[b].strip() - while (not _bounded_string(ab) or - (len(ab) > 2 and - ab[0] == ab[1] == ab[2] and - ab[-2] != ab[0] and - ab[-3] != ab[0])): - a[b] = a[b] + ',' + a[b + 1] - ab = a[b].strip() - if b < len(a) - 2: - a = a[:b + 1] + a[b + 2:] - else: - a = a[:b + 1] - b += 1 - else: - al = list(a[1:-1]) - a = [] - openarr = 0 - j = 0 - for i in _range(len(al)): - if al[i] == '[': - openarr += 1 - elif al[i] == ']': - openarr -= 1 - elif al[i] == ',' and not openarr: - a.append(''.join(al[j:i])) - j = i + 1 - a.append(''.join(al[j:])) - for i in _range(len(a)): - a[i] = a[i].strip() - if a[i] != '': - nval, ntype = _load_value(a[i], _dict) - if atype: - if ntype != atype: - raise ValueError("Not a homogeneous array") - else: - atype = ntype - retval.append(nval) - return retval - - -def dump(o, f): - """Writes out dict as toml to a file - - Args: - o: Object to dump into toml - f: File descriptor where the toml should be stored - - Returns: - String containing the toml corresponding to dictionary - - Raises: - TypeError: When anything other than file descriptor is passed - """ - - if not f.write: - raise TypeError("You can only dump an object to a file descriptor") - d = dumps(o) - f.write(d) - return d - - -def dumps(o, preserve=False): - """Stringifies input dict as toml - - Args: - o: Object to dump into toml - - preserve: Boolean parameter. If true, preserve inline tables. - - Returns: - String containing the toml corresponding to dict - """ - - retval = "" - addtoretval, sections = _dump_sections(o, "") - retval += addtoretval - while sections != {}: - newsections = {} - for section in sections: - addtoretval, addtosections = _dump_sections(sections[section], - section, preserve) - if addtoretval or (not addtoretval and not addtosections): - if retval and retval[-2:] != "\n\n": - retval += "\n" - retval += "[" + section + "]\n" - if addtoretval: - retval += addtoretval - for s in addtosections: - newsections[section + "." + s] = addtosections[s] - sections = newsections - return retval - - -def _dump_sections(o, sup, preserve=False): - retstr = "" - if sup != "" and sup[-1] != ".": - sup += '.' - retdict = o.__class__() - arraystr = "" - for section in o: - section = unicode(section) - qsection = section - if not re.match(r'^[A-Za-z0-9_-]+$', section): - if '"' in section: - qsection = "'" + section + "'" - else: - qsection = '"' + section + '"' - if not isinstance(o[section], dict): - arrayoftables = False - if isinstance(o[section], list): - for a in o[section]: - if isinstance(a, dict): - arrayoftables = True - if arrayoftables: - for a in o[section]: - arraytabstr = "\n" - arraystr += "[[" + sup + qsection + "]]\n" - s, d = _dump_sections(a, sup + qsection) - if s: - if s[0] == "[": - arraytabstr += s - else: - arraystr += s - while d != {}: - newd = {} - for dsec in d: - s1, d1 = _dump_sections(d[dsec], sup + qsection + - "." + dsec) - if s1: - arraytabstr += ("[" + sup + qsection + "." + - dsec + "]\n") - arraytabstr += s1 - for s1 in d1: - newd[dsec + "." + s1] = d1[s1] - d = newd - arraystr += arraytabstr - else: - if o[section] is not None: - retstr += (qsection + " = " + - unicode(_dump_value(o[section])) + '\n') - elif preserve and isinstance(o[section], InlineTableDict): - retstr += (qsection + " = " + _dump_inline_table(o[section])) - else: - retdict[qsection] = o[section] - retstr += arraystr - return (retstr, retdict) - - -def _dump_inline_table(section): - """Preserve inline table in its compact syntax instead of expanding - into subsection. - - https://github.com/toml-lang/toml#user-content-inline-table - """ - retval = "" - if isinstance(section, dict): - val_list = [] - for k, v in section.items(): - val = _dump_inline_table(v) - val_list.append(k + " = " + val) - retval += "{ " + ", ".join(val_list) + " }\n" - return retval - else: - return unicode(_dump_value(section)) - - -def _dump_value(v): - dump_funcs = { - str: _dump_str, - unicode: _dump_str, - list: _dump_list, - int: lambda v: v, - bool: lambda v: unicode(v).lower(), - float: _dump_float, - datetime.datetime: lambda v: v.isoformat().replace('+00:00', 'Z'), - } - # Lookup function corresponding to v's type - dump_fn = dump_funcs.get(type(v)) - if dump_fn is None and hasattr(v, '__iter__'): - dump_fn = dump_funcs[list] - # Evaluate function (if it exists) else return v - return dump_fn(v) if dump_fn is not None else dump_funcs[str](v) - - -def _dump_str(v): - if sys.version_info < (3,) and hasattr(v, 'decode') and isinstance(v, str): - v = v.decode('utf-8') - v = "%r" % v - if v[0] == 'u': - v = v[1:] - singlequote = v.startswith("'") - if singlequote or v.startswith('"'): - v = v[1:-1] - if singlequote: - v = v.replace("\\'", "'") - v = v.replace('"', '\\"') - v = v.split("\\x") - while len(v) > 1: - i = -1 - if not v[0]: - v = v[1:] - v[0] = v[0].replace("\\\\", "\\") - # No, I don't know why != works and == breaks - joinx = v[0][i] != "\\" - while v[0][:i] and v[0][i] == "\\": - joinx = not joinx - i -= 1 - if joinx: - joiner = "x" - else: - joiner = "u00" - v = [v[0] + joiner + v[1]] + v[2:] - return unicode('"' + v[0] + '"') - - -def _dump_list(v): - retval = "[" - for u in v: - retval += " " + unicode(_dump_value(u)) + "," - retval += "]" - return retval - - -def _dump_float(v): - return "{0:.16}".format(v).replace("e+0", "e+").replace("e-0", "e-") diff --git a/pipenv/vendor/tomlkit/__init__.py b/pipenv/vendor/tomlkit/__init__.py index 23d4ef74..89e4cf59 100644 --- a/pipenv/vendor/tomlkit/__init__.py +++ b/pipenv/vendor/tomlkit/__init__.py @@ -22,4 +22,4 @@ from .api import value from .api import ws -__version__ = "0.4.4" +__version__ = "0.4.6" diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index bb3696d9..37014921 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -4,7 +4,6 @@ from ._compat import decode from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey from .items import AoT -from .items import Bool from .items import Comment from .items import Item from .items import Key @@ -525,3 +524,21 @@ class Container(dict): return NotImplemented return self.value == other + + def _getstate(self, protocol): + return (self._parsed,) + + def __reduce__(self): + return self.__reduce_ex__(2) + + def __reduce_ex__(self, protocol): + return ( + self.__class__, + self._getstate(protocol), + (self._map, self._body, self._parsed), + ) + + def __setstate__(self, state): + self._map = state[0] + self._body = state[1] + self._parsed = state[2] diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py index d889a924..46ee938b 100644 --- a/pipenv/vendor/tomlkit/exceptions.py +++ b/pipenv/vendor/tomlkit/exceptions.py @@ -85,7 +85,7 @@ class InvalidCharInStringError(ParseError): """ def __init__(self, line, col, char): # type: (int, int, str) -> None - message = "Invalid character '{}' in string".format(char) + message = "Invalid character {} in string".format(repr(char)) super(InvalidCharInStringError, self).__init__(line, col, message=message) diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 26f24701..abece020 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -17,6 +17,11 @@ from ._compat import decode from ._compat import unicode from ._utils import escape_string +if PY2: + from functools32 import lru_cache +else: + from functools import lru_cache + def item(value, _parent=None): from .container import Container @@ -75,18 +80,45 @@ def item(value, _parent=None): class StringType(Enum): - + # Single Line Basic SLB = '"' + # Multi Line Basic MLB = '"""' + # Single Line Literal SLL = "'" + # Multi Line Literal MLL = "'''" + @property + @lru_cache(maxsize=None) + def unit(self): # type: () -> str + return self.value[0] + + @lru_cache(maxsize=None) + def is_basic(self): # type: () -> bool + return self in {StringType.SLB, StringType.MLB} + + @lru_cache(maxsize=None) def is_literal(self): # type: () -> bool return self in {StringType.SLL, StringType.MLL} + @lru_cache(maxsize=None) + def is_singleline(self): # type: () -> bool + return self in {StringType.SLB, StringType.SLL} + + @lru_cache(maxsize=None) def is_multiline(self): # type: () -> bool return self in {StringType.MLB, StringType.MLL} + @lru_cache(maxsize=None) + def toggle(self): # type: () -> StringType + return { + StringType.SLB: StringType.MLB, + StringType.MLB: StringType.SLB, + StringType.SLL: StringType.MLL, + StringType.MLL: StringType.SLL, + }[self] + class Trivia: """ @@ -158,7 +190,10 @@ class Key: return hash(self.key) def __eq__(self, other): # type: (Key) -> bool - return self.key == other.key + if isinstance(other, Key): + return self.key == other.key + + return self.key == other def __str__(self): # type: () -> str return self.as_string() @@ -205,6 +240,15 @@ class Item(object): return self + def _getstate(self, protocol=3): + return (self._trivia,) + + def __reduce__(self): + return self.__reduce_ex__(2) + + def __reduce_ex__(self, protocol): + return self.__class__, self._getstate(protocol) + class Whitespace(Item): """ @@ -240,6 +284,9 @@ class Whitespace(Item): def __repr__(self): # type: () -> str return "<{} {}>".format(self.__class__.__name__, repr(self._s)) + def _getstate(self, protocol=3): + return self._s, self._fixed + class Comment(Item): """ @@ -273,7 +320,7 @@ class Integer(int, Item): self._raw = raw self._sign = False - if re.match("^[+\-]\d+$", raw): + if re.match(r"^[+\-]\d+$", raw): self._sign = True @property @@ -322,6 +369,9 @@ class Integer(int, Item): return Integer(result, self._trivia, raw) + def _getstate(self, protocol=3): + return int(self), self._trivia, self._raw + class Float(float, Item): """ @@ -337,7 +387,7 @@ class Float(float, Item): self._raw = raw self._sign = False - if re.match("^[+\-].+$", raw): + if re.match(r"^[+\-].+$", raw): self._sign = True @property @@ -386,6 +436,9 @@ class Float(float, Item): return Float(result, self._trivia, raw) + def _getstate(self, protocol=3): + return float(self), self._trivia, self._raw + class Bool(Item): """ @@ -408,13 +461,16 @@ class Bool(Item): def as_string(self): # type: () -> str return str(self._value).lower() + def _getstate(self, protocol=3): + return self._value, self._trivia -class DateTime(datetime, Item): + +class DateTime(Item, datetime): """ A datetime literal. """ - def __new__(cls, value, *_): # type: (datetime, ...) -> datetime + def __new__(cls, value, *_): # type: (..., datetime, ...) -> datetime return datetime.__new__( cls, value.year, @@ -458,13 +514,29 @@ class DateTime(datetime, Item): return DateTime(result, self._trivia, raw) + def _getstate(self, protocol=3): + return ( + datetime( + self.year, + self.month, + self.day, + self.hour, + self.minute, + self.second, + self.microsecond, + self.tzinfo, + ), + self._trivia, + self._raw, + ) -class Date(date, Item): + +class Date(Item, date): """ A date literal. """ - def __new__(cls, value, *_): # type: (date, ...) -> date + def __new__(cls, value, *_): # type: (..., date, ...) -> date return date.__new__(cls, value.year, value.month, value.day) def __init__(self, _, trivia, raw): # type: (date, Trivia, str) -> None @@ -498,8 +570,11 @@ class Date(date, Item): return Date(result, self._trivia, raw) + def _getstate(self, protocol=3): + return (datetime(self.year, self.month, self.day), self._trivia, self._raw) -class Time(time, Item): + +class Time(Item, time): """ A time literal. """ @@ -525,6 +600,13 @@ class Time(time, Item): def as_string(self): # type: () -> str return self._raw + def _getstate(self, protocol=3): + return ( + time(self.hour, self.minute, self.second, self.microsecond, self.tzinfo), + self._trivia, + self._raw, + ) + class Array(Item, list): """ @@ -623,6 +705,9 @@ class Array(Item, list): def __repr__(self): return str(self) + def _getstate(self, protocol=3): + return self._value, self._trivia + class Table(Item, dict): """ @@ -637,7 +722,7 @@ class Table(Item, dict): is_super_table=False, name=None, display_name=None, - ): # type: (tomlkit.container.Container, Trivia, bool) -> None + ): # type: (tomlkit.container.Container, Trivia, bool, ...) -> None super(Table, self).__init__(trivia) self.name = name @@ -790,6 +875,16 @@ class Table(Item, dict): def __repr__(self): return super(Table, self).__repr__() + def _getstate(self, protocol=3): + return ( + self._value, + self._trivia, + self._is_aot_element, + self._is_super_table, + self.name, + self.display_name, + ) + class InlineTable(Item, dict): """ @@ -924,6 +1019,9 @@ class InlineTable(Item, dict): def __repr__(self): return super(InlineTable, self).__repr__() + def _getstate(self, protocol=3): + return (self._value, self._trivia) + class String(unicode, Item): """ @@ -965,6 +1063,9 @@ class String(unicode, Item): def _new(self, result): return String(self._t, result, result, self._trivia) + def _getstate(self, protocol=3): + return self._t, unicode(self), self._original, self._trivia + class AoT(Item, list): """ @@ -974,7 +1075,7 @@ class AoT(Item, list): def __init__( self, body, name=None, parsed=False ): # type: (List[Table], Optional[str]) -> None - self.name = None + self.name = name self._body = [] self._parsed = parsed @@ -1025,6 +1126,9 @@ class AoT(Item, list): def __repr__(self): # type: () -> str return "".format(self.value) + def _getstate(self, protocol=3): + return self._body, self.name, self._parsed + class Null(Item): """ @@ -1044,3 +1148,6 @@ class Null(Item): def as_string(self): # type: () -> str return "" + + def _getstate(self, protocol=3): + return tuple() diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py index 45c8ee8c..7971d9a2 100644 --- a/pipenv/vendor/tomlkit/parser.py +++ b/pipenv/vendor/tomlkit/parser.py @@ -75,7 +75,7 @@ class Parser: else: return self._src[self._marker : self._idx] - def inc(self): # type: () -> bool + def inc(self, exception=None): # type: () -> bool """ Increments the parser if the end of the input has not been reached. Returns whether or not it was able to advance. @@ -88,15 +88,17 @@ class Parser: self._idx = len(self._src) self._current = TOMLChar("\0") - return False + if not exception: + return False + raise exception - def inc_n(self, n): # type: (int) -> bool + def inc_n(self, n, exception=None): # type: (int) -> bool """ Increments the parser by n characters if the end of the input has not been reached. """ for _ in range(n): - if not self.inc(): + if not self.inc(exception=exception): return False return True @@ -336,7 +338,7 @@ class Parser: self.mark() break - elif c in " \t\r,": + elif c in " \t\r": self.inc() else: raise self.parse_error(UnexpectedCharError, (c)) @@ -669,140 +671,143 @@ class Parser: return def _parse_literal_string(self): # type: () -> Item - return self._parse_string("'") + return self._parse_string(StringType.SLL) def _parse_basic_string(self): # type: () -> Item - return self._parse_string('"') + return self._parse_string(StringType.SLB) + + def _parse_escaped_char(self, multiline): + if multiline and self._current.is_ws(): + # When the last non-whitespace character on a line is + # a \, it will be trimmed along with all whitespace + # (including newlines) up to the next non-whitespace + # character or closing delimiter. + # """\ + # hello \ + # world""" + tmp = "" + while self._current.is_ws(): + tmp += self._current + # consume the whitespace, EOF here is an issue + # (middle of string) + self.inc(exception=UnexpectedEofError) + continue + + # the escape followed by whitespace must have a newline + # before any other chars + if "\n" not in tmp: + raise self.parse_error(InvalidCharInStringError, (self._current,)) + + return "" + + if self._current in _escaped: + c = _escaped[self._current] + + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) + + return c + + if self._current in {"u", "U"}: + # this needs to be a unicode + u, ue = self._peek_unicode(self._current == "U") + if u is not None: + # consume the U char and the unicode value + self.inc_n(len(ue) + 1) + + return u + + raise self.parse_error(InvalidCharInStringError, (self._current,)) def _parse_string(self, delim): # type: (str) -> Item - multiline = False + delim = StringType(delim) + assert delim.is_singleline() + + # only keep parsing for string if the current character matches the delim + if self._current != delim.unit: + raise ValueError("Expecting a {!r} character".format(delim)) + + # consume the opening/first delim, EOF here is an issue + # (middle of string or middle of delim) + self.inc(exception=UnexpectedEofError) + + if self._current == delim.unit: + # consume the closing/second delim, we do not care if EOF occurs as + # that would simply imply an empty single line string + if not self.inc() or self._current != delim.unit: + # Empty string + return String(delim, "", "", Trivia()) + + # consume the third delim, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) + + delim = delim.toggle() # convert delim to multi delim + + self.mark() # to extract the original string with whitespace and all value = "" - if delim == "'": - str_type = StringType.SLL - else: - str_type = StringType.SLB + # A newline immediately following the opening delimiter will be trimmed. + if delim.is_multiline() and self._current == "\n": + # consume the newline, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) - # Skip opening delim - if not self.inc(): - return self.parse_error(UnexpectedEofError) - - if self._current == delim: - self.inc() - - if self._current == delim: - multiline = True - if delim == "'": - str_type = StringType.MLL - else: - str_type = StringType.MLB - - if not self.inc(): - return self.parse_error(UnexpectedEofError) - else: - # Empty string - return String(str_type, "", "", Trivia()) - - self.mark() - if self._current == "\n": - # The first new line should be discarded - self.inc() - - previous = None - escaped = False + escaped = False # whether the previous key was ESCAPE while True: - if ( - previous != "\\" - or previous == "\\" - and (escaped or str_type.is_literal()) - ) and self._current == delim: - val = self.extract() + if delim.is_singleline() and self._current.is_nl(): + # single line cannot have actual newline characters + raise self.parse_error(InvalidCharInStringError, (self._current,)) + elif not escaped and self._current == delim.unit: + # try to process current as a closing delim + original = self.extract() - if multiline: - stop = True - for _ in range(3): - if self._current != delim: + close = "" + if delim.is_multiline(): + # try consuming three delims as this would mean the end of + # the string + for last in [False, False, True]: + if self._current != delim.unit: # Not a triple quote, leave in result as-is. - stop = False - - # Adding back the quote character - value += delim + # Adding back the characters we already consumed + value += close + close = "" # clear the close break - self.inc() # TODO: Handle EOF + close += delim.unit - if not stop: + # consume this delim, EOF here is only an issue if this + # is not the third (last) delim character + self.inc(exception=UnexpectedEofError if not last else None) + + if not close: # if there is no close characters, keep parsing continue else: + close = delim.unit + + # consume the closing delim, we do not care if EOF occurs as + # that would simply imply the end of self._src self.inc() - return String(str_type, value, val, Trivia()) + return String(delim, value, original, Trivia()) + elif delim.is_basic() and escaped: + # attempt to parse the current char as an escaped value, an exception + # is raised if this fails + value += self._parse_escaped_char(delim.is_multiline()) + + # no longer escaped + escaped = False + elif delim.is_basic() and self._current == "\\": + # the next char is being escaped + escaped = True + + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) else: - if previous == "\\" and self._current.is_ws() and multiline: - while self._current.is_ws(): - previous = self._current + # this is either a literal string where we keep everything as is, + # or this is not a special escaped char in a basic string + value += self._current - self.inc() - continue - - if self._current == delim: - continue - - if previous == "\\": - if self._current == "\\" and not escaped: - if not str_type.is_literal(): - escaped = True - else: - value += self._current - - previous = self._current - - if not self.inc(): - raise self.parse_error(UnexpectedEofError) - - continue - elif self._current in _escaped and not escaped: - if not str_type.is_literal(): - value = value[:-1] - value += _escaped[self._current] - else: - value += self._current - elif self._current in {"u", "U"} and not escaped: - # Maybe unicode - u, ue = self._peek_unicode(self._current == "U") - if u is not None: - value = value[:-1] - value += u - self.inc_n(len(ue)) - else: - if not escaped and not str_type.is_literal(): - raise self.parse_error( - InvalidCharInStringError, (self._current,) - ) - - value += self._current - else: - if not escaped and not str_type.is_literal(): - raise self.parse_error( - InvalidCharInStringError, (self._current,) - ) - - value += self._current - - if self._current.is_ws() and multiline and not escaped: - continue - else: - value += self._current - - if escaped: - escaped = False - - previous = self._current - if not self.inc(): - raise self.parse_error(UnexpectedEofError) - - if previous == "\\" and self._current.is_ws() and multiline: - value = value[:-1] + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) def _parse_table( self, parent_name=None diff --git a/pipenv/vendor/tomlkit/toml_char.py b/pipenv/vendor/tomlkit/toml_char.py index 8a3bf9e1..92f1a9c9 100644 --- a/pipenv/vendor/tomlkit/toml_char.py +++ b/pipenv/vendor/tomlkit/toml_char.py @@ -1,7 +1,13 @@ import string +from ._compat import PY2 from ._compat import unicode +if PY2: + from functools32 import lru_cache +else: + from functools import lru_cache + class TOMLChar(unicode): def __init__(self, c): @@ -10,36 +16,42 @@ class TOMLChar(unicode): if len(self) > 1: raise ValueError("A TOML character must be of length 1") + @lru_cache(maxsize=None) def is_bare_key_char(self): # type: () -> bool """ Whether the character is a valid bare key name or not. """ return self in string.ascii_letters + string.digits + "-" + "_" + @lru_cache(maxsize=None) def is_kv_sep(self): # type: () -> bool """ Whether the character is a valid key/value separator ot not. """ return self in "= \t" + @lru_cache(maxsize=None) def is_int_float_char(self): # type: () -> bool """ Whether the character if a valid integer or float value character or not. """ return self in string.digits + "+" + "-" + "_" + "." + "e" + @lru_cache(maxsize=None) def is_ws(self): # type: () -> bool """ Whether the character is a whitespace character or not. """ return self in " \t\r\n" + @lru_cache(maxsize=None) def is_nl(self): # type: () -> bool """ Whether the character is a new line character or not. """ return self in "\n\r" + @lru_cache(maxsize=None) def is_spaces(self): # type: () -> bool """ Whether the character is a space or not diff --git a/pipenv/vendor/urllib3/__init__.py b/pipenv/vendor/urllib3/__init__.py old mode 100755 new mode 100644 index 4bd533b5..75725167 --- a/pipenv/vendor/urllib3/__init__.py +++ b/pipenv/vendor/urllib3/__init__.py @@ -23,16 +23,11 @@ from .util.retry import Retry # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.23' +__version__ = '1.24' __all__ = ( 'HTTPConnectionPool', diff --git a/pipenv/vendor/urllib3/_collections.py b/pipenv/vendor/urllib3/_collections.py old mode 100755 new mode 100644 index 6e36b84e..34f23811 --- a/pipenv/vendor/urllib3/_collections.py +++ b/pipenv/vendor/urllib3/_collections.py @@ -14,10 +14,7 @@ except ImportError: # Platform-specific: No threads available pass -try: # Python 2.7+ - from collections import OrderedDict -except ImportError: - from .packages.ordered_dict import OrderedDict +from collections import OrderedDict from .exceptions import InvalidHeader from .packages.six import iterkeys, itervalues, PY3 diff --git a/pipenv/vendor/urllib3/connection.py b/pipenv/vendor/urllib3/connection.py old mode 100755 new mode 100644 index a03b573f..02b36654 --- a/pipenv/vendor/urllib3/connection.py +++ b/pipenv/vendor/urllib3/connection.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import datetime import logging import os -import sys import socket from socket import error as SocketError, timeout as SocketTimeout import warnings @@ -78,9 +77,6 @@ class HTTPConnection(_HTTPConnection, object): - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool` - ``source_address``: Set the source address for the current connection. - - .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x - - ``socket_options``: Set specific options on the underlying socket. If not specified, then defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy. @@ -108,21 +104,13 @@ class HTTPConnection(_HTTPConnection, object): if six.PY3: # Python 3 kw.pop('strict', None) - # Pre-set source_address in case we have an older Python like 2.6. + # Pre-set source_address. self.source_address = kw.get('source_address') - if sys.version_info < (2, 7): # Python 2.6 - # _HTTPConnection on Python 2.6 will balk at this keyword arg, but - # not newer versions. We can still use it when creating a - # connection though, so we pop it *after* we have saved it as - # self.source_address. - kw.pop('source_address', None) - #: The socket options provided by the user. If no options are #: provided, we use the default options. self.socket_options = kw.pop('socket_options', self.default_socket_options) - # Superclass also sets self.source_address in Python 2.7+. _HTTPConnection.__init__(self, *args, **kw) @property @@ -183,10 +171,7 @@ class HTTPConnection(_HTTPConnection, object): def _prepare_conn(self, conn): self.sock = conn - # the _tunnel_host attribute was added in python 2.6.3 (via - # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do - # not have them. - if getattr(self, '_tunnel_host', None): + if self._tunnel_host: # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() # Mark this connection as not reusable @@ -217,13 +202,13 @@ class HTTPConnection(_HTTPConnection, object): self.endheaders() if body is not None: - stringish_types = six.string_types + (six.binary_type,) + stringish_types = six.string_types + (bytes,) if isinstance(body, stringish_types): body = (body,) for chunk in body: if not chunk: continue - if not isinstance(chunk, six.binary_type): + if not isinstance(chunk, bytes): chunk = chunk.encode('utf8') len_str = hex(len(chunk))[2:] self.send(len_str.encode('utf-8')) @@ -242,7 +227,7 @@ class HTTPSConnection(HTTPConnection): def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - ssl_context=None, **kw): + ssl_context=None, server_hostname=None, **kw): HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) @@ -250,6 +235,7 @@ class HTTPSConnection(HTTPConnection): self.key_file = key_file self.cert_file = cert_file self.ssl_context = ssl_context + self.server_hostname = server_hostname # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) @@ -270,6 +256,7 @@ class HTTPSConnection(HTTPConnection): keyfile=self.key_file, certfile=self.cert_file, ssl_context=self.ssl_context, + server_hostname=self.server_hostname ) @@ -312,12 +299,9 @@ class VerifiedHTTPSConnection(HTTPSConnection): def connect(self): # Add certificate verification conn = self._new_conn() - hostname = self.host - if getattr(self, '_tunnel_host', None): - # _tunnel_host was added in Python 2.6.3 - # (See: http://hg.python.org/cpython/rev/0f57b30a152f) + if self._tunnel_host: self.sock = conn # Calls self._set_hostport(), so self.host is # self._tunnel_host below. @@ -328,6 +312,10 @@ class VerifiedHTTPSConnection(HTTPSConnection): # Override the host with the one we're requesting data from. hostname = self._tunnel_host + server_hostname = hostname + if self.server_hostname is not None: + server_hostname = self.server_hostname + is_time_off = datetime.date.today() < RECENT_DATE if is_time_off: warnings.warn(( @@ -352,7 +340,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): certfile=self.cert_file, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, - server_hostname=hostname, + server_hostname=server_hostname, ssl_context=context) if self.assert_fingerprint: @@ -373,7 +361,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): 'for details.)'.format(hostname)), SubjectAltNameWarning ) - _match_hostname(cert, self.assert_hostname or hostname) + _match_hostname(cert, self.assert_hostname or server_hostname) self.is_verified = ( context.verify_mode == ssl.CERT_REQUIRED or diff --git a/pipenv/vendor/urllib3/connectionpool.py b/pipenv/vendor/urllib3/connectionpool.py old mode 100755 new mode 100644 index 8fcb0bce..f7a8f193 --- a/pipenv/vendor/urllib3/connectionpool.py +++ b/pipenv/vendor/urllib3/connectionpool.py @@ -89,7 +89,7 @@ class ConnectionPool(object): # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 -_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) +_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK} class HTTPConnectionPool(ConnectionPool, RequestMethods): @@ -313,7 +313,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Catch possible read timeouts thrown as SSL errors. If not the # case, rethrow the original. We need to do this because of: # http://bugs.python.org/issue10272 - if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 + if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python < 2.7.4 raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) def _make_request(self, conn, method, url, timeout=_Default, chunked=False, @@ -375,7 +375,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): try: try: # Python 2.7, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) - except TypeError: # Python 2.6 and older, Python 3 + except TypeError: # Python 3 try: httplib_response = conn.getresponse() except Exception as e: @@ -801,17 +801,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): Establish tunnel connection early, because otherwise httplib would improperly set Host: header to proxy's IP:port. """ - # Python 2.7+ - try: - set_tunnel = conn.set_tunnel - except AttributeError: # Platform-specific: Python 2.6 - set_tunnel = conn._set_tunnel - - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older - set_tunnel(self._proxy_host, self.port) - else: - set_tunnel(self._proxy_host, self.port, self.proxy_headers) - + conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) conn.connect() def _new_conn(self): diff --git a/pipenv/vendor/urllib3/contrib/__init__.py b/pipenv/vendor/urllib3/contrib/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_appengine_environ.py b/pipenv/vendor/urllib3/contrib/_appengine_environ.py new file mode 100644 index 00000000..f3e00942 --- /dev/null +++ b/pipenv/vendor/urllib3/contrib/_appengine_environ.py @@ -0,0 +1,30 @@ +""" +This module provides means to detect the App Engine environment. +""" + +import os + + +def is_appengine(): + return (is_local_appengine() or + is_prod_appengine() or + is_prod_appengine_mvms()) + + +def is_appengine_sandbox(): + return is_appengine() and not is_prod_appengine_mvms() + + +def is_local_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Development/' in os.environ['SERVER_SOFTWARE']) + + +def is_prod_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and + not is_prod_appengine_mvms()) + + +def is_prod_appengine_mvms(): + return os.environ.get('GAE_VM', False) == 'true' diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/__init__.py b/pipenv/vendor/urllib3/contrib/_securetransport/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py b/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py b/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/appengine.py b/pipenv/vendor/urllib3/contrib/appengine.py old mode 100755 new mode 100644 index 66922e06..2952f114 --- a/pipenv/vendor/urllib3/contrib/appengine.py +++ b/pipenv/vendor/urllib3/contrib/appengine.py @@ -39,8 +39,8 @@ urllib3 on Google App Engine: """ from __future__ import absolute_import +import io import logging -import os import warnings from ..packages.six.moves.urllib.parse import urljoin @@ -53,11 +53,11 @@ from ..exceptions import ( SSLError ) -from ..packages.six import BytesIO from ..request import RequestMethods from ..response import HTTPResponse from ..util.timeout import Timeout from ..util.retry import Retry +from . import _appengine_environ try: from google.appengine.api import urlfetch @@ -239,7 +239,7 @@ class AppEngineManager(RequestMethods): original_response = HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), msg=urlfetch_resp.header_msg, headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, @@ -247,7 +247,7 @@ class AppEngineManager(RequestMethods): ) return HTTPResponse( - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, original_response=original_response, @@ -280,26 +280,10 @@ class AppEngineManager(RequestMethods): return retries -def is_appengine(): - return (is_local_appengine() or - is_prod_appengine() or - is_prod_appengine_mvms()) +# Alias methods from _appengine_environ to maintain public API interface. - -def is_appengine_sandbox(): - return is_appengine() and not is_prod_appengine_mvms() - - -def is_local_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Development/' in os.environ['SERVER_SOFTWARE']) - - -def is_prod_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and - not is_prod_appengine_mvms()) - - -def is_prod_appengine_mvms(): - return os.environ.get('GAE_VM', False) == 'true' +is_appengine = _appengine_environ.is_appengine +is_appengine_sandbox = _appengine_environ.is_appengine_sandbox +is_local_appengine = _appengine_environ.is_local_appengine +is_prod_appengine = _appengine_environ.is_prod_appengine +is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms diff --git a/pipenv/vendor/urllib3/contrib/ntlmpool.py b/pipenv/vendor/urllib3/contrib/ntlmpool.py old mode 100755 new mode 100644 index 642e99ed..8ea127c5 --- a/pipenv/vendor/urllib3/contrib/ntlmpool.py +++ b/pipenv/vendor/urllib3/contrib/ntlmpool.py @@ -43,8 +43,7 @@ class NTLMConnectionPool(HTTPSConnectionPool): log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', self.num_connections, self.host, self.authurl) - headers = {} - headers['Connection'] = 'Keep-Alive' + headers = {'Connection': 'Keep-Alive'} req_header = 'Authorization' resp_header = 'www-authenticate' diff --git a/pipenv/vendor/urllib3/contrib/pyopenssl.py b/pipenv/vendor/urllib3/contrib/pyopenssl.py old mode 100755 new mode 100644 index 4d4b1aff..7c0e9465 --- a/pipenv/vendor/urllib3/contrib/pyopenssl.py +++ b/pipenv/vendor/urllib3/contrib/pyopenssl.py @@ -163,6 +163,9 @@ def _dnsname_to_stdlib(name): from ASCII bytes. We need to idna-encode that string to get it back, and then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). + + If the name cannot be idna-encoded then we return None signalling that + the name given should be skipped. """ def idna_encode(name): """ @@ -172,14 +175,19 @@ def _dnsname_to_stdlib(name): """ import idna - for prefix in [u'*.', u'.']: - if name.startswith(prefix): - name = name[len(prefix):] - return prefix.encode('ascii') + idna.encode(name) - return idna.encode(name) + try: + for prefix in [u'*.', u'.']: + if name.startswith(prefix): + name = name[len(prefix):] + return prefix.encode('ascii') + idna.encode(name) + return idna.encode(name) + except idna.core.IDNAError: + return None name = idna_encode(name) - if sys.version_info >= (3, 0): + if name is None: + return None + elif sys.version_info >= (3, 0): name = name.decode('utf-8') return name @@ -223,9 +231,10 @@ def get_subj_alt_name(peer_cert): # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8 # decoded. This is pretty frustrating, but that's what the standard library # does with certificates, and so we need to attempt to do the same. + # We also want to skip over names which cannot be idna encoded. names = [ - ('DNS', _dnsname_to_stdlib(name)) - for name in ext.get_values_for_type(x509.DNSName) + ('DNS', name) for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) + if name is not None ] names.extend( ('IP Address', str(name)) diff --git a/pipenv/vendor/urllib3/contrib/securetransport.py b/pipenv/vendor/urllib3/contrib/securetransport.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/socks.py b/pipenv/vendor/urllib3/contrib/socks.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/exceptions.py b/pipenv/vendor/urllib3/exceptions.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/fields.py b/pipenv/vendor/urllib3/fields.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/filepost.py b/pipenv/vendor/urllib3/filepost.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/__init__.py b/pipenv/vendor/urllib3/packages/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/backports/__init__.py b/pipenv/vendor/urllib3/packages/backports/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/backports/makefile.py b/pipenv/vendor/urllib3/packages/backports/makefile.py old mode 100755 new mode 100644 index 75b80dcf..740db377 --- a/pipenv/vendor/urllib3/packages/backports/makefile.py +++ b/pipenv/vendor/urllib3/packages/backports/makefile.py @@ -16,7 +16,7 @@ def backport_makefile(self, mode="r", buffering=None, encoding=None, """ Backport of ``socket.makefile`` from Python 3.5. """ - if not set(mode) <= set(["r", "w", "b"]): + if not set(mode) <= {"r", "w", "b"}: raise ValueError( "invalid mode %r (only r, w, b allowed)" % (mode,) ) diff --git a/pipenv/vendor/urllib3/packages/ordered_dict.py b/pipenv/vendor/urllib3/packages/ordered_dict.py deleted file mode 100755 index 4479363c..00000000 --- a/pipenv/vendor/urllib3/packages/ordered_dict.py +++ /dev/null @@ -1,259 +0,0 @@ -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. -# Copyright 2009 Raymond Hettinger, released under the MIT License. -# http://code.activestate.com/recipes/576693/ -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) diff --git a/pipenv/vendor/urllib3/packages/six.py b/pipenv/vendor/urllib3/packages/six.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py b/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py b/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py old mode 100755 new mode 100644 index 1fd42f38..d6e66c01 --- a/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py +++ b/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py @@ -9,8 +9,7 @@ import sys # ipaddress has been backported to 2.6+ in pypi. If it is installed on the # system, use it to handle IPAddress ServerAltnames (this was added in # python-3.5) otherwise only do DNS matching. This allows -# backports.ssl_match_hostname to continue to be used all the way back to -# python-2.4. +# backports.ssl_match_hostname to continue to be used in Python 2.7. try: import ipaddress except ImportError: diff --git a/pipenv/vendor/urllib3/poolmanager.py b/pipenv/vendor/urllib3/poolmanager.py old mode 100755 new mode 100644 index 506a3c9b..fe5491cf --- a/pipenv/vendor/urllib3/poolmanager.py +++ b/pipenv/vendor/urllib3/poolmanager.py @@ -47,6 +47,7 @@ _key_fields = ( 'key__socks_options', # dict 'key_assert_hostname', # bool or string 'key_assert_fingerprint', # str + 'key_server_hostname', #str ) #: The namedtuple class used to construct keys for the connection pool. diff --git a/pipenv/vendor/urllib3/request.py b/pipenv/vendor/urllib3/request.py old mode 100755 new mode 100644 index 1be33341..8f2f44bb --- a/pipenv/vendor/urllib3/request.py +++ b/pipenv/vendor/urllib3/request.py @@ -36,7 +36,7 @@ class RequestMethods(object): explicitly. """ - _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + _encode_url_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS'} def __init__(self, headers=None): self.headers = headers or {} diff --git a/pipenv/vendor/urllib3/response.py b/pipenv/vendor/urllib3/response.py old mode 100755 new mode 100644 index 9873cb94..f0cfbb54 --- a/pipenv/vendor/urllib3/response.py +++ b/pipenv/vendor/urllib3/response.py @@ -11,7 +11,7 @@ from .exceptions import ( BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked, IncompleteRead, InvalidHeader ) -from .packages.six import string_types as basestring, binary_type, PY3 +from .packages.six import string_types as basestring, PY3 from .packages.six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head @@ -23,7 +23,7 @@ class DeflateDecoder(object): def __init__(self): self._first_try = True - self._data = binary_type() + self._data = b'' self._obj = zlib.decompressobj() def __getattr__(self, name): @@ -69,7 +69,7 @@ class GzipDecoder(object): return getattr(self._obj, name) def decompress(self, data): - ret = binary_type() + ret = b'' if self._state == GzipDecoderState.SWALLOW_DATA or not data: return ret while True: @@ -90,7 +90,31 @@ class GzipDecoder(object): self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) +class MultiDecoder(object): + """ + From RFC7231: + If one or more encodings have been applied to a representation, the + sender that applied the encodings MUST generate a Content-Encoding + header field that lists the content codings in the order in which + they were applied. + """ + + def __init__(self, modes): + self._decoders = [_get_decoder(m.strip()) for m in modes.split(',')] + + def flush(self): + return self._decoders[0].flush() + + def decompress(self, data): + for d in reversed(self._decoders): + data = d.decompress(data) + return data + + def _get_decoder(mode): + if ',' in mode: + return MultiDecoder(mode) + if mode == 'gzip': return GzipDecoder() @@ -159,7 +183,7 @@ class HTTPResponse(io.IOBase): self.msg = msg self._request_url = request_url - if body and isinstance(body, (basestring, binary_type)): + if body and isinstance(body, (basestring, bytes)): self._body = body self._pool = pool @@ -283,8 +307,13 @@ class HTTPResponse(io.IOBase): # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None and content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + if self._decoder is None: + if content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + elif ',' in content_encoding: + encodings = [e.strip() for e in content_encoding.split(',') if e.strip() in self.CONTENT_DECODERS] + if len(encodings): + self._decoder = _get_decoder(content_encoding) def _decode(self, data, decode_content, flush_decoder): """ diff --git a/pipenv/vendor/urllib3/util/__init__.py b/pipenv/vendor/urllib3/util/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/connection.py b/pipenv/vendor/urllib3/util/connection.py old mode 100755 new mode 100644 index 5cf488f4..5ad70b2f --- a/pipenv/vendor/urllib3/util/connection.py +++ b/pipenv/vendor/urllib3/util/connection.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import socket from .wait import NoWayToWaitForSocketError, wait_for_read +from ..contrib import _appengine_environ def is_connection_dropped(conn): # Platform-specific @@ -105,6 +106,13 @@ def _has_ipv6(host): sock = None has_ipv6 = False + # App Engine doesn't support IPV6 sockets and actually has a quota on the + # number of sockets that can be used, so just early out here instead of + # creating a socket needlessly. + # See https://github.com/urllib3/urllib3/issues/1446 + if _appengine_environ.is_appengine_sandbox(): + return False + if socket.has_ipv6: # has_ipv6 returns true if cPython was compiled with IPv6 support. # It does not tell us if the system has IPv6 support enabled. To diff --git a/pipenv/vendor/urllib3/util/queue.py b/pipenv/vendor/urllib3/util/queue.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/request.py b/pipenv/vendor/urllib3/util/request.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/response.py b/pipenv/vendor/urllib3/util/response.py old mode 100755 new mode 100644 index 67cf730a..3d548648 --- a/pipenv/vendor/urllib3/util/response.py +++ b/pipenv/vendor/urllib3/util/response.py @@ -59,8 +59,14 @@ def assert_header_parsing(headers): get_payload = getattr(headers, 'get_payload', None) unparsed_data = None - if get_payload: # Platform-specific: Python 3. - unparsed_data = get_payload() + if get_payload: + # get_payload is actually email.message.Message.get_payload; + # we're only interested in the result if it's not a multipart message + if not headers.is_multipart(): + payload = get_payload() + + if isinstance(payload, (bytes, str)): + unparsed_data = payload if defects or unparsed_data: raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data) diff --git a/pipenv/vendor/urllib3/util/retry.py b/pipenv/vendor/urllib3/util/retry.py old mode 100755 new mode 100644 index 7ad3dc66..e7d0abd6 --- a/pipenv/vendor/urllib3/util/retry.py +++ b/pipenv/vendor/urllib3/util/retry.py @@ -115,7 +115,7 @@ class Retry(object): (most errors are resolved immediately by a second try without a delay). urllib3 will sleep for:: - {backoff factor} * (2 ^ ({number of total retries} - 1)) + {backoff factor} * (2 ** ({number of total retries} - 1)) seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer diff --git a/pipenv/vendor/urllib3/util/ssl_.py b/pipenv/vendor/urllib3/util/ssl_.py old mode 100755 new mode 100644 index 2893752a..24ee26d6 --- a/pipenv/vendor/urllib3/util/ssl_.py +++ b/pipenv/vendor/urllib3/util/ssl_.py @@ -56,9 +56,8 @@ except ImportError: OP_NO_COMPRESSION = 0x20000 -# Python 2.7 and earlier didn't have inet_pton on non-Linux -# so we fallback on inet_aton in those cases. This means that -# we can only detect IPv4 addresses in this case. +# Python 2.7 doesn't have inet_pton on non-Linux so we fallback on inet_aton in +# those cases. This means that we can only detect IPv4 addresses in this case. if hasattr(socket, 'inet_pton'): inet_pton = socket.inet_pton else: @@ -67,7 +66,7 @@ else: import ipaddress def inet_pton(_, host): - if isinstance(host, six.binary_type): + if isinstance(host, bytes): host = host.decode('ascii') return ipaddress.ip_address(host) @@ -115,10 +114,7 @@ try: except ImportError: import sys - class SSLContext(object): # Platform-specific: Python 2 & 3.1 - supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or - (3, 2) <= sys.version_info) - + class SSLContext(object): # Platform-specific: Python 2 def __init__(self, protocol_version): self.protocol = protocol_version # Use default values from a real SSLContext @@ -141,12 +137,6 @@ except ImportError: raise SSLError("CA directories not supported in older Pythons") def set_ciphers(self, cipher_suite): - if not self.supports_set_ciphers: - raise TypeError( - 'Your version of Python does not support setting ' - 'a custom cipher suite. Please upgrade to Python ' - '2.7, 3.2, or later if you need this functionality.' - ) self.ciphers = cipher_suite def wrap_socket(self, socket, server_hostname=None, server_side=False): @@ -167,10 +157,7 @@ except ImportError: 'ssl_version': self.protocol, 'server_side': server_side, } - if self.supports_set_ciphers: # Platform-specific: Python 2.7+ - return wrap_socket(socket, ciphers=self.ciphers, **kwargs) - else: # Platform-specific: Python 2.6 - return wrap_socket(socket, **kwargs) + return wrap_socket(socket, ciphers=self.ciphers, **kwargs) def assert_fingerprint(cert, fingerprint): @@ -291,9 +278,6 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, context.options |= options - if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 - context.set_ciphers(ciphers or DEFAULT_CIPHERS) - context.verify_mode = cert_reqs if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 # We do our own verification, including fingerprints and alternative @@ -316,8 +300,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, A pre-made :class:`SSLContext` object. If none is provided, one will be created using :func:`create_urllib3_context`. :param ciphers: - A string of ciphers we wish the client to support. This is not - supported on Python 2.6 as the ssl module does not support it. + A string of ciphers we wish the client to support. :param ca_cert_dir: A directory containing CA certificates in multiple separate files, as supported by OpenSSL's -CApath flag or the capath argument to @@ -334,7 +317,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if ca_certs or ca_cert_dir: try: context.load_verify_locations(ca_certs, ca_cert_dir) - except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + except IOError as e: # Platform-specific: Python 2.7 raise SSLError(e) # Py33 raises FileNotFoundError which subclasses OSError # These are not equivalent unless we check the errno attribute @@ -378,7 +361,7 @@ def is_ipaddress(hostname): :param str hostname: Hostname to examine. :return: True if the hostname is an IP address, False otherwise. """ - if six.PY3 and isinstance(hostname, six.binary_type): + if six.PY3 and isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. hostname = hostname.decode('ascii') diff --git a/pipenv/vendor/urllib3/util/timeout.py b/pipenv/vendor/urllib3/util/timeout.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/url.py b/pipenv/vendor/urllib3/util/url.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/wait.py b/pipenv/vendor/urllib3/util/wait.py old mode 100755 new mode 100644 index fa686eff..4db71baf --- a/pipenv/vendor/urllib3/util/wait.py +++ b/pipenv/vendor/urllib3/util/wait.py @@ -43,9 +43,6 @@ if sys.version_info >= (3, 5): else: # Old and broken Pythons. def _retry_on_intr(fn, timeout): - if timeout is not None and timeout <= 0: - return fn(timeout) - if timeout is None: deadline = float("inf") else: @@ -117,7 +114,7 @@ def _have_working_poll(): # from libraries like eventlet/greenlet. try: poll_obj = select.poll() - poll_obj.poll(0) + _retry_on_intr(poll_obj.poll, 0) except (AttributeError, OSError): return False else: diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 7a574b9f..8812d2ae 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -14,28 +14,28 @@ first==2.0.1 iso8601==0.1.12 jinja2==2.10 markupsafe==1.0 -parse==1.8.4 +parse==1.9.0 pathlib2==2.3.2 scandir==1.9 pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.2 -requests==2.19.1 +pythonfinder==1.1.3 +requests==2.20.0 chardet==3.0.4 idna==2.7 - urllib3==1.23 - certifi==2018.8.24 -requirementslib==1.1.9 + urllib3==1.24 + certifi==2018.10.15 +requirementslib==1.1.10 attrs==18.2.0 distlib==0.2.8 packaging==18.0 pyparsing==2.2.2 pytoml==0.1.19 plette==0.2.2 - tomlkit==0.4.4 -shellingham==1.2.6 + tomlkit==0.4.6 +shellingham==1.2.7 six==1.11.0 semver==2.8.1 shutilwhich==1.1.0 @@ -48,3 +48,5 @@ enum34==1.1.6 yaspin==0.14.0 cerberus==1.2 git+https://github.com/sarugaku/passa.git@master#egg=passa +cursor==1.2.0 +backports.functools_lru_cache==1.5 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index fe6f884c..881985d2 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -8,12 +8,14 @@ from .contextmanagers import ( open_file, temp_environ, temp_path, + spinner, ) from .misc import load_path, partialclass, run, shell_escape -from .path import mkdir_p, rmtree +from .path import mkdir_p, rmtree, create_tracked_tempdir +from .spin import VistirSpinner, create_spinner -__version__ = '0.1.7' +__version__ = '0.1.8' __all__ = [ @@ -31,4 +33,8 @@ __all__ = [ "TemporaryDirectory", "NamedTemporaryFile", "partialmethod", + "spinner", + "VistirSpinner", + "create_spinner", + "create_tracked_tempdir" ] diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index 0c865fe6..eab87908 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import errno import os import sys import warnings @@ -16,18 +17,21 @@ __all__ = [ "finalize", "partialmethod", "JSONDecodeError", + "FileNotFoundError", "ResourceWarning", "FileNotFoundError", "fs_str", + "lru_cache", "TemporaryDirectory", "NamedTemporaryFile", ] if sys.version_info >= (3, 5): from pathlib import Path - + from functools import lru_cache else: from pathlib2 import Path + from pipenv.vendor.backports.functools_lru_cache import lru_cache if sys.version_info < (3, 3): from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size @@ -57,16 +61,18 @@ if six.PY2: pass class FileNotFoundError(IOError): - pass + """No such file or directory""" + + def __init__(self, *args, **kwargs): + self.errno = errno.ENOENT + super(FileNotFoundError, self).__init__(*args, **kwargs) else: from builtins import ResourceWarning, FileNotFoundError - class ResourceWarning(ResourceWarning): - pass - class FileNotFoundError(FileNotFoundError): - pass +if not sys.warnoptions: + warnings.simplefilter("default", ResourceWarning) class TemporaryDirectory(object): diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 80f1f897..70f95c59 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import io import os import stat import sys @@ -13,7 +14,9 @@ from .compat import NamedTemporaryFile, Path from .path import is_file_url, is_valid_url, path_to_url, url_to_path -__all__ = ["temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file"] +__all__ = [ + "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner" +] # Borrowed from Pew. @@ -77,6 +80,66 @@ def cd(path): os.chdir(prev_cwd) +@contextmanager +def dummy_spinner(spin_type, text, **kwargs): + class FakeClass(object): + def __init__(self, text=""): + self.text = text + + def fail(self, exitcode=1, text=None): + if text: + print(text) + raise SystemExit(exitcode, text) + + def ok(self, text): + print(text) + return 0 + + def write(self, text): + print(text) + + myobj = FakeClass(text) + yield myobj + + +@contextmanager +def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): + """Get a spinner object or a dummy spinner to wrap a context. + + :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) + :param str start_text: Text to start off the spinner with (default: {None}) + :param dict handler_map: Handler map for signals to be handled gracefully (default: {None}) + :param bool nospin: If true, use the dummy spinner (default: {False}) + :return: A spinner object which can be manipulated while alive + :rtype: :class:`~vistir.spin.VistirSpinner` + + Raises: + RuntimeError -- Raised if the spinner extra is not installed + """ + + from .spin import create_spinner + spinner_func = create_spinner + if nospin is False: + try: + import yaspin + except ImportError: + raise RuntimeError( + "Failed to import spinner! Reinstall vistir with command:" + " pip install --upgrade vistir[spinner]" + ) + else: + spinner_name = None + if not start_text: + start_text = "Running..." + with spinner_func( + spinner_name=spinner_name, + text=start_text, + handler_map=handler_map, + nospin=nospin, + ) as _spinner: + yield _spinner + + @contextmanager def atomic_open_for_write(target, binary=False, newline=None, encoding=None): """Atomically open `target` for writing. @@ -192,8 +255,11 @@ def open_file(link, session=None): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(link)) else: - with open(local_path, "rb") as local_file: + try: + local_file = io.open(local_path, "rb") yield local_file + finally: + local_file.close() else: # Remote URL headers = {"Accept-Encoding": "identity"} diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 44607a98..f42b4ad1 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -2,19 +2,24 @@ from __future__ import absolute_import, unicode_literals import json +import logging import locale import os import subprocess import sys from collections import OrderedDict -from contextlib import contextmanager from functools import partial import six from .cmdparse import Script from .compat import Path, fs_str, partialmethod +from .contextmanagers import spinner as spinner + +if os.name != "nt": + class WindowsError(OSError): + pass __all__ = [ @@ -30,6 +35,22 @@ __all__ = [ ] +def _get_logger(name=None, level="ERROR"): + if not name: + name = __name__ + if isinstance(level, six.string_types): + level = getattr(logging, level.upper()) + logger = logging.getLogger(name) + logger.setLevel(level) + formatter = logging.Formatter( + "%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s" + ) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + def shell_escape(cmd): """Escape strings for use in :func:`~subprocess.Popen` and :func:`run`. @@ -75,9 +96,11 @@ def dedup(iterable): return iter(OrderedDict.fromkeys(iterable)) -def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True): +def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=True): from distutils.spawn import find_executable + if not env: + env = {} command = find_executable(script.command) options = { "env": env, @@ -102,7 +125,7 @@ def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True) try: return subprocess.Popen(cmd, **options) except WindowsError as e: - if e.winerror != 193: + if getattr(e, "winerror", 9999) != 193: raise options["shell"] = True # Try shell mode to use Windows's file association for file launch. @@ -111,7 +134,7 @@ def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True) def _create_subprocess( cmd, - env={}, + env=None, block=True, return_object=False, cwd=os.curdir, @@ -120,6 +143,8 @@ def _create_subprocess( combine_stderr=False, display_limit=200 ): + if not env: + env = {} try: c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr) @@ -128,11 +153,13 @@ def _create_subprocess( raise if not block: c.stdin.close() + log_level = "DEBUG" if verbose else "WARN" + logger = _get_logger(cmd._parts[0], level=log_level) output = [] err = [] spinner_orig_text = "" if spinner: - spinner_orig_text = spinner.text + spinner_orig_text = getattr(spinner, "text", "") streams = { "stdout": c.stdout, "stderr": c.stderr @@ -147,7 +174,7 @@ def _create_subprocess( line = to_text(stream.readline()) if not line: continue - line = line.rstrip() + line = to_text("{0}".format(line.rstrip())) if outstream == "stderr": stderr_line = line else: @@ -155,15 +182,24 @@ def _create_subprocess( if not (stdout_line or stderr_line): break if stderr_line: - err.append(line) + err.append(stderr_line) + if verbose: + if spinner: + spinner.write_err(fs_str(stderr_line)) + else: + logger.error(stderr_line) if stdout_line: output.append(stdout_line) display_line = stdout_line if len(stdout_line) > display_limit: display_line = "{0}...".format(stdout_line[:display_limit]) if verbose: - spinner.write(display_line) - spinner.text = "{0} {1}".format(spinner_orig_text, display_line) + if spinner: + spinner.write(fs_str(display_line)) + else: + logger.debug(display_line) + if spinner: + spinner.text = fs_str("{0} {1}".format(spinner_orig_text, display_line)) continue try: c.wait() @@ -175,18 +211,22 @@ def _create_subprocess( if spinner: if c.returncode > 0: spinner.fail("Failed...cleaning up...") - spinner.text = "Complete!" - spinner.ok("✔") + else: + spinner.text = "Complete!" + if not os.name == "nt": + spinner.ok("✔") + else: + spinner.ok() c.out = "\n".join(output) c.err = "\n".join(err) if err else "" else: c.out, c.err = c.communicate() + if not block: + c.wait() + c.out = fs_str("{0}".format(c.out)) if c.out else fs_str("") + c.err = fs_str("{0}".format(c.err)) if c.err else fs_str("") if not return_object: - if not block: - c.wait() - out = c.out if c.out else "" - err = c.err if c.err else "" - return out.strip(), err.strip() + return c.out.strip(), c.err.strip() return c @@ -198,7 +238,7 @@ def run( cwd=None, verbose=False, nospin=False, - spinner=None, + spinner_name=None, combine_stderr=True, display_limit=200 ): @@ -211,7 +251,7 @@ def run( :param str cwd: Current working directory contect to use for spawning the subprocess. :param bool verbose: Whether to print stdout in real time when non-blocking. :param bool nospin: Whether to disable the cli spinner. - :param str spinner: The name of the spinner to use if enabled, defaults to bouncingBar + :param str spinner_name: The name of the spinner to use if enabled, defaults to bouncingBar :param bool combine_stderr: Optionally merge stdout and stderr in the subprocess, false if nonblocking. :param int dispay_limit: The max width of output lines to display when using a spinner. :returns: A 2-tuple of (output, error) or a :class:`subprocess.Popen` object. @@ -221,8 +261,10 @@ def run( this functionality. """ - if not env: - env = os.environ.copy() + _env = os.environ.copy() + if env: + _env.update(env) + env = _env if six.PY2: fs_encode = partial(to_bytes, encoding=locale_encoding) _env = {fs_encode(k): fs_encode(v) for k, v in os.environ.items()} @@ -230,8 +272,8 @@ def run( _env[fs_encode(key)] = fs_encode(val) else: _env = {k: fs_str(v) for k, v in os.environ.items()} - if not spinner: - spinner = "bouncingBar" + if not spinner_name: + spinner_name = "bouncingBar" if six.PY2: if isinstance(cmd, six.string_types): cmd = cmd.encode("utf-8") @@ -241,48 +283,7 @@ def run( cmd = Script.parse(cmd) if block or not return_object: combine_stderr = False - sigmap = {} - if nospin is False: - try: - import signal - from yaspin import yaspin - from yaspin import spinners - from yaspin.signal_handlers import fancy_handler - except ImportError: - raise RuntimeError( - "Failed to import spinner! Reinstall vistir with command:" - " pip install --upgrade vistir[spinner]" - ) - else: - animation = getattr(spinners.Spinners, spinner) - sigmap = { - signal.SIGINT: fancy_handler - } - if os.name == "nt": - sigmap.update({ - signal.CTRL_C_EVENT: fancy_handler, - signal.CTRL_BREAK_EVENT: fancy_handler - }) - spinner_func = yaspin - else: - - @contextmanager - def spinner_func(spin_type, text, **kwargs): - class FakeClass(object): - def __init__(self, text=""): - self.text = text - - def ok(self, text): - return - - def write(self, text): - print(text) - - myobj = FakeClass(text) - yield myobj - - animation = None - with spinner_func(animation, sigmap=sigmap, text="Running...") as sp: + with spinner(spinner_name=spinner_name, start_text="Running...", nospin=nospin) as sp: return _create_subprocess( cmd, env=_env, diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 166282e8..ce7ecee0 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -15,8 +15,7 @@ import six from six.moves import urllib_parse from six.moves.urllib import request as urllib_request -from .compat import Path, _fs_encoding, TemporaryDirectory -from .misc import locale_encoding, to_bytes, to_text +from .compat import Path, _fs_encoding, TemporaryDirectory, ResourceWarning __all__ = [ @@ -74,6 +73,7 @@ def normalize_drive(path): identified with either upper or lower cased drive names. The case is always converted to uppercase because it seems to be preferred. """ + from .misc import to_text if os.name != "nt" or not isinstance(path, six.string_types): return path @@ -95,6 +95,7 @@ def path_to_url(path): >>> path_to_url("/home/user/code/myrepo/myfile.zip") 'file:///home/user/code/myrepo/myfile.zip' """ + from .misc import to_text, to_bytes if not path: return path @@ -108,6 +109,7 @@ def url_to_path(url): Follows logic taken from pip's equivalent function """ + from .misc import to_bytes assert is_file_url(url), "Only file: urls can be converted to local paths" _, netloc, path, _, _ = urllib_parse.urlsplit(url) # Netlocs are UNC paths @@ -120,14 +122,16 @@ def url_to_path(url): def is_valid_url(url): """Checks if a given string is an url""" + from .misc import to_text if not url: return url - pieces = urllib_parse.urlparse(url) + pieces = urllib_parse.urlparse(to_text(url)) return all([pieces.scheme, pieces.netloc]) def is_file_url(url): """Returns true if the given url is a file url""" + from .misc import to_text if not url: return False if not isinstance(url, six.string_types): @@ -144,6 +148,7 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ + from .misc import to_bytes fn = to_bytes(fn, encoding="utf-8") if os.path.exists(fn): return bool(os.stat(fn).st_mode & stat.S_IREAD) and not os.access(fn, os.W_OK) @@ -158,7 +163,8 @@ def mkdir_p(newdir, mode=0o777): :raises: OSError if a file is encountered along the way """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ - newdir = abspathu(to_bytes(newdir, "utf-8")) + from .misc import to_bytes, to_text + newdir = to_bytes(newdir, "utf-8") if os.path.exists(newdir): if not os.path.isdir(newdir): raise OSError( @@ -166,17 +172,17 @@ def mkdir_p(newdir, mode=0o777): newdir ) ) - pass else: - head, tail = os.path.split(newdir) + head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) # Make sure the tail doesn't point to the asame place as the head - tail_and_head_match = os.path.relpath(tail, start=os.path.basename(head)) == "." + curdir = to_bytes(".", encoding="utf-8") + tail_and_head_match = os.path.relpath(tail, start=os.path.basename(head)) == curdir if tail and not tail_and_head_match and not os.path.isdir(newdir): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): raise OSError( "A file with the same name as the desired dir, '{0}', already exists.".format( - newdir + to_text(newdir, encoding="utf-8") ) ) os.makedirs(os.path.join(head, tail), mode) @@ -210,9 +216,11 @@ def create_tracked_tempdir(*args, **kwargs): The return value is the path to the created directory. """ + tempdir = TemporaryDirectory(*args, **kwargs) TRACKED_TEMPORARY_DIRECTORIES.append(tempdir) atexit.register(tempdir.cleanup) + warnings.simplefilter("default", ResourceWarning) return tempdir.name @@ -223,6 +231,7 @@ def set_write_bit(fn): :param str fn: The target filename or path """ + from .misc import to_bytes, locale_encoding fn = to_bytes(fn, encoding=locale_encoding) if not os.path.exists(fn): return @@ -243,10 +252,17 @@ def rmtree(directory, ignore_errors=False): Setting `ignore_errors=True` may cause this to silently fail to delete the path """ + from .misc import locale_encoding, to_bytes directory = to_bytes(directory, encoding=locale_encoding) - shutil.rmtree( - directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly - ) + try: + shutil.rmtree( + directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly + ) + except (IOError, OSError) as exc: + # Ignore removal failures where the file doesn't exist + if exc.errno == errno.ENOENT: + pass + raise def handle_remove_readonly(func, path, exc): @@ -263,35 +279,41 @@ def handle_remove_readonly(func, path, exc): :func:`set_write_bit` on the target path and try again. """ # Check for read-only attribute - from .compat import ResourceWarning + if six.PY2: + from .compat import ResourceWarning + from .misc import to_bytes + PERM_ERRORS = (errno.EACCES, errno.EPERM) default_warning_message = ( "Unable to remove file due to permissions restriction: {!r}" ) # split the initial exception out into its type, exception, and traceback exc_type, exc_exception, exc_tb = exc - path = to_bytes(path) + path = to_bytes(path, encoding="utf-8") if is_readonly_path(path): # Apply write permission and call original function set_write_bit(path) try: func(path) except (OSError, IOError) as e: - if e.errno in [errno.EACCES, errno.EPERM]: - warnings.warn( - default_warning_message.format( - to_text(path, encoding=locale_encoding) - ), ResourceWarning - ) + if e.errno in PERM_ERRORS: + warnings.warn(default_warning_message.format(path), ResourceWarning) return - if exc_exception.errno in [errno.EACCES, errno.EPERM]: - warnings.warn( - default_warning_message.format(to_text(path)), - ResourceWarning - ) - return - - raise + if exc_exception.errno in PERM_ERRORS: + set_write_bit(path) + try: + func(path) + except (OSError, IOError) as e: + if e.errno in PERM_ERRORS: + warnings.warn(default_warning_message.format(path), ResourceWarning) + elif e.errno == errno.ENOENT: # File already gone + return + else: + raise + return + else: + raise + raise exc def walk_up(bottom): @@ -356,6 +378,7 @@ def get_converted_relative_path(path, relative_to=None): >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder') '.' """ + from .misc import to_text, to_bytes # noqa if not relative_to: relative_to = os.getcwdu() if six.PY2 else os.getcwd() diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py new file mode 100644 index 00000000..d2cddd79 --- /dev/null +++ b/pipenv/vendor/vistir/spin.py @@ -0,0 +1,149 @@ +# -*- coding=utf-8 -*- +import os +import signal +import sys + +from .termcolors import colored +from .compat import fs_str + +import cursor +import functools +try: + import yaspin +except ImportError: + yaspin = None + Spinners = None +else: + from yaspin.spinners import Spinners + +handler = None +if yaspin and os.name == "nt": + handler = yaspin.signal_handlers.default_handler +elif yaspin and os.name != "nt": + handler = yaspin.signal_handlers.fancy_handler + +CLEAR_LINE = chr(27) + "[K" + + +class DummySpinner(object): + def __init__(self, text="", **kwargs): + self.text = text + + def __enter__(self): + if self.text: + self.write(self.text) + return self + + def __exit__(self, exc_type, exc_val, traceback): + if not exc_type: + self.ok() + else: + self.write_err(traceback) + return False + + def fail(self, exitcode=1, text=None): + if text: + self.write_err(text) + raise SystemExit(exitcode, text) + + def ok(self, text=None): + if text: + self.write(self.text) + return 0 + + def write(self, text=None): + if text: + line = fs_str("{0}\n".format(text)) + sys.stdout.write(line) + + def write_err(self, text=None): + if text: + line = fs_str("{0}\n".format(text)) + sys.stderr.write(line) + + +base_obj = yaspin.core.Yaspin if yaspin is not None else DummySpinner + + +class VistirSpinner(base_obj): + def __init__(self, *args, **kwargs): + """Get a spinner object or a dummy spinner to wrap a context. + + Keyword Arguments: + :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) + :param str start_text: Text to start off the spinner with (default: {None}) + :param dict handler_map: Handler map for signals to be handled gracefully (default: {None}) + :param bool nospin: If true, use the dummy spinner (default: {False}) + """ + + self.handler = handler + sigmap = {} + if handler: + sigmap.update({ + signal.SIGINT: handler, + signal.SIGTERM: handler + }) + handler_map = kwargs.pop("handler_map", {}) + if os.name == "nt": + sigmap[signal.SIGBREAK] = handler + else: + sigmap[signal.SIGALRM] = handler + if handler_map: + sigmap.update(handler_map) + spinner_name = kwargs.pop("spinner_name", "bouncingBar") + text = kwargs.pop("start_text", "") + " " + kwargs.pop("text", "") + if not text: + text = "Running..." + kwargs["sigmap"] = sigmap + kwargs["spinner"] = getattr(Spinners, spinner_name, Spinners.bouncingBar) + super(VistirSpinner, self).__init__(*args, **kwargs) + self.is_dummy = bool(yaspin is None) + + def fail(self, exitcode=1, *args, **kwargs): + super(VistirSpinner, self).fail(**kwargs) + + def ok(self, *args, **kwargs): + super(VistirSpinner, self).ok(*args, **kwargs) + + def write(self, *args, **kwargs): + super(VistirSpinner, self).write(*args, **kwargs) + + def write_err(self, text): + """Write error text in the terminal without breaking the spinner.""" + + sys.stderr.write("\r") + self._clear_err() + text = fs_str("{0}\n".format(text)) + sys.stderr.write(text) + + def _compose_color_func(self): + fn = functools.partial( + colored, + color=self._color, + on_color=self._on_color, + attrs=list(self._attrs), + ) + return fn + + @staticmethod + def _hide_cursor(): + cursor.hide() + + @staticmethod + def _show_cursor(): + cursor.show() + + @staticmethod + def _clear_err(): + sys.stderr.write(CLEAR_LINE) + + @staticmethod + def _clear_line(): + sys.stdout.write(CLEAR_LINE) + + +def create_spinner(*args, **kwargs): + nospin = kwargs.pop("nospin", False) + if nospin: + return DummySpinner(*args, **kwargs) + return VistirSpinner(*args, **kwargs) diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py new file mode 100644 index 00000000..6f3ad32c --- /dev/null +++ b/pipenv/vendor/vistir/termcolors.py @@ -0,0 +1,116 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals +import colorama +import os + + +ATTRIBUTES = dict( + list(zip([ + 'bold', + 'dark', + '', + 'underline', + 'blink', + '', + 'reverse', + 'concealed' + ], + list(range(1, 9)) + )) + ) +del ATTRIBUTES[''] + + +HIGHLIGHTS = dict( + list(zip([ + 'on_grey', + 'on_red', + 'on_green', + 'on_yellow', + 'on_blue', + 'on_magenta', + 'on_cyan', + 'on_white' + ], + list(range(40, 48)) + )) + ) + + +COLORS = dict( + list(zip([ + 'grey', + 'red', + 'green', + 'yellow', + 'blue', + 'magenta', + 'cyan', + 'white', + ], + list(range(30, 38)) + )) + ) + + +RESET = colorama.Style.RESET_ALL + + +def colored(text, color=None, on_color=None, attrs=None): + """Colorize text using a reimplementation of the colorizer from + https://github.com/pavdmyt/yaspin so that it works on windows. + + Available text colors: + red, green, yellow, blue, magenta, cyan, white. + + Available text highlights: + on_red, on_green, on_yellow, on_blue, on_magenta, on_cyan, on_white. + + Available attributes: + bold, dark, underline, blink, reverse, concealed. + + Example: + colored('Hello, World!', 'red', 'on_grey', ['blue', 'blink']) + colored('Hello, World!', 'green') + """ + if os.getenv('ANSI_COLORS_DISABLED') is None: + style = "NORMAL" + if 'bold' in attrs: + style = "BRIGHT" + attrs.remove('bold') + if color is not None: + text = text = "%s%s%s%s%s" % ( + getattr(colorama.Fore, color), + getattr(colorama.Style, style), + text, + colorama.Fore.RESET, + colorama.Style.NORMAL, + ) + + if on_color is not None: + text = "%s%s%s%s" % ( + getattr(colorama.Back, color), + text, + colorama.Back.RESET, + colorama.Style.NORMAL, + ) + + if attrs is not None: + fmt_str = "%s[%%dm%%s%s[9m" % ( + chr(27), + chr(27) + ) + for attr in attrs: + text = fmt_str % (ATTRIBUTES[attr], text) + + text += RESET + return text + + +def cprint(text, color=None, on_color=None, attrs=None, **kwargs): + """Print colorize text. + + It accepts arguments of print function. + """ + + print((colored(text, color, on_color, attrs)), **kwargs) From 4c8617237ccdf53bcf87607479afbd9e8cee99bb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 21 Oct 2018 23:54:30 -0400 Subject: [PATCH 012/218] Update requirementslib, requests and vistir Signed-off-by: Dan Ryan --- pipenv/vendor/requests/__init__.py | 17 +-- pipenv/vendor/requests/__version__.py | 4 +- pipenv/vendor/requests/adapters.py | 27 +++-- pipenv/vendor/requests/api.py | 20 ++-- pipenv/vendor/requests/auth.py | 4 +- pipenv/vendor/requests/compat.py | 3 +- pipenv/vendor/requests/cookies.py | 31 ++--- pipenv/vendor/requests/help.py | 3 +- pipenv/vendor/requests/hooks.py | 4 +- pipenv/vendor/requests/models.py | 17 +-- pipenv/vendor/requests/sessions.py | 52 ++++++--- pipenv/vendor/requests/status_codes.py | 2 +- pipenv/vendor/requests/utils.py | 19 ++-- .../requirementslib/models/dependencies.py | 11 +- .../vendor/requirementslib/models/lockfile.py | 49 ++++++-- .../vendor/requirementslib/models/pipfile.py | 78 +++++++++++-- .../requirementslib/models/requirements.py | 4 +- pipenv/vendor/requirementslib/utils.py | 106 ++++++++++++++++++ pipenv/vendor/vistir/contextmanagers.py | 2 +- pipenv/vendor/vistir/misc.py | 55 ++++++++- 20 files changed, 393 insertions(+), 115 deletions(-) diff --git a/pipenv/vendor/requests/__init__.py b/pipenv/vendor/requests/__init__.py index a5b3c9c3..bc168ee5 100644 --- a/pipenv/vendor/requests/__init__.py +++ b/pipenv/vendor/requests/__init__.py @@ -22,7 +22,7 @@ usage: ... or POST: >>> payload = dict(key1='value1', key2='value2') - >>> r = requests.post('http://httpbin.org/post', data=payload) + >>> r = requests.post('https://httpbin.org/post', data=payload) >>> print(r.text) { ... @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.23 + # urllib3 >= 1.21.1, <= 1.24 assert major == 1 assert minor >= 21 - assert minor <= 23 + assert minor <= 24 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] @@ -79,14 +79,14 @@ def _check_cryptography(cryptography_version): return if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warning = 'Old version of cryptography ({}) may cause slowdown.'.format(cryptography_version) warnings.warn(warning, RequestsDependencyWarning) # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) except (AssertionError, ValueError): - warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported " + warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " "version!".format(urllib3.__version__, chardet.__version__), RequestsDependencyWarning) @@ -123,12 +123,7 @@ from .exceptions import ( # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/pipenv/vendor/requests/__version__.py b/pipenv/vendor/requests/__version__.py index ef61ec0f..be8a45fe 100644 --- a/pipenv/vendor/requests/__version__.py +++ b/pipenv/vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.19.1' -__build__ = 0x021901 +__version__ = '2.20.0' +__build__ = 0x022000 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/pipenv/vendor/requests/adapters.py b/pipenv/vendor/requests/adapters.py index a4b02842..fa4d9b3c 100644 --- a/pipenv/vendor/requests/adapters.py +++ b/pipenv/vendor/requests/adapters.py @@ -26,6 +26,7 @@ from urllib3.exceptions import ProtocolError from urllib3.exceptions import ReadTimeoutError from urllib3.exceptions import SSLError as _SSLError from urllib3.exceptions import ResponseError +from urllib3.exceptions import LocationValueError from .models import Response from .compat import urlparse, basestring @@ -35,7 +36,8 @@ from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL, + InvalidURL) from .auth import _basic_auth_str try: @@ -127,8 +129,7 @@ class HTTPAdapter(BaseAdapter): self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in - self.__attrs__) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because @@ -224,7 +225,7 @@ class HTTPAdapter(BaseAdapter): if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) + "invalid path: {}".format(cert_loc)) conn.cert_reqs = 'CERT_REQUIRED' @@ -246,10 +247,10 @@ class HTTPAdapter(BaseAdapter): conn.key_file = None if conn.cert_file and not os.path.exists(conn.cert_file): raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(conn.cert_file)) + "invalid path: {}".format(conn.cert_file)) if conn.key_file and not os.path.exists(conn.key_file): raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(conn.key_file)) + "invalid path: {}".format(conn.key_file)) def build_response(self, req, resp): """Builds a :class:`Response ` object from a urllib3 @@ -378,7 +379,7 @@ class HTTPAdapter(BaseAdapter): when subclassing the :class:`HTTPAdapter `. - :param proxies: The url of the proxy being used for this request. + :param proxy: The url of the proxy being used for this request. :rtype: dict """ headers = {} @@ -407,7 +408,10 @@ class HTTPAdapter(BaseAdapter): :rtype: requests.Response """ - conn = self.get_connection(request.url, proxies) + try: + conn = self.get_connection(request.url, proxies) + except LocationValueError as e: + raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) @@ -421,7 +425,7 @@ class HTTPAdapter(BaseAdapter): timeout = TimeoutSauce(connect=connect, read=read) except ValueError as e: # this may raise a string formatting error. - err = ("Invalid timeout {0}. Pass a (connect, read) " + err = ("Invalid timeout {}. Pass a (connect, read) " "timeout tuple, or a single float to set " "both timeouts to the same value".format(timeout)) raise ValueError(err) @@ -471,11 +475,10 @@ class HTTPAdapter(BaseAdapter): # Receive the response from the server try: - # For Python 2.7+ versions, use buffering of HTTP - # responses + # For Python 2.7, use buffering of HTTP responses r = low_conn.getresponse(buffering=True) except TypeError: - # For compatibility with Python 2.6 versions and back + # For compatibility with Python 3.3+ r = low_conn.getresponse() resp = HTTPResponse.from_httplib( diff --git a/pipenv/vendor/requests/api.py b/pipenv/vendor/requests/api.py index a2cc84d7..abada96d 100644 --- a/pipenv/vendor/requests/api.py +++ b/pipenv/vendor/requests/api.py @@ -18,8 +18,10 @@ def request(method, url, **kwargs): :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. @@ -47,7 +49,7 @@ def request(method, url, **kwargs): Usage:: >>> import requests - >>> req = requests.request('GET', 'http://httpbin.org/get') + >>> req = requests.request('GET', 'https://httpbin.org/get') """ @@ -62,7 +64,8 @@ def get(url, params=None, **kwargs): r"""Sends a GET request. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response @@ -102,7 +105,8 @@ def post(url, data=None, json=None, **kwargs): r"""Sends a POST request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -116,7 +120,8 @@ def put(url, data=None, **kwargs): r"""Sends a PUT request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -130,7 +135,8 @@ def patch(url, data=None, **kwargs): r"""Sends a PATCH request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object diff --git a/pipenv/vendor/requests/auth.py b/pipenv/vendor/requests/auth.py index 4ae45947..bdde51c7 100644 --- a/pipenv/vendor/requests/auth.py +++ b/pipenv/vendor/requests/auth.py @@ -38,7 +38,7 @@ def _basic_auth_str(username, password): if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(username), category=DeprecationWarning, @@ -48,7 +48,7 @@ def _basic_auth_str(username, password): if not isinstance(password, basestring): warnings.warn( "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(password), category=DeprecationWarning, diff --git a/pipenv/vendor/requests/compat.py b/pipenv/vendor/requests/compat.py index 6b9c6fac..c44b35ef 100644 --- a/pipenv/vendor/requests/compat.py +++ b/pipenv/vendor/requests/compat.py @@ -43,9 +43,8 @@ if is_py2: import cookielib from Cookie import Morsel from StringIO import StringIO - from collections import Callable, Mapping, MutableMapping + from collections import Callable, Mapping, MutableMapping, OrderedDict - from urllib3.packages.ordered_dict import OrderedDict builtin_str = str bytes = str diff --git a/pipenv/vendor/requests/cookies.py b/pipenv/vendor/requests/cookies.py index 50883a84..56fccd9c 100644 --- a/pipenv/vendor/requests/cookies.py +++ b/pipenv/vendor/requests/cookies.py @@ -444,20 +444,21 @@ def create_cookie(name, value, **kwargs): By default, the pair of `name` and `value` will be set for the domain '' and sent on every request (this is sometimes called a "supercookie"). """ - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False,) + result = { + 'version': 0, + 'name': name, + 'value': value, + 'port': None, + 'domain': '', + 'path': '/', + 'secure': False, + 'expires': None, + 'discard': True, + 'comment': None, + 'comment_url': None, + 'rest': {'HttpOnly': None}, + 'rfc2109': False, + } badargs = set(kwargs) - set(result) if badargs: @@ -511,6 +512,7 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): :param cookiejar: (optional) A cookiejar to add the cookies to. :param overwrite: (optional) If False, will not replace cookies already in the jar with new ones. + :rtype: CookieJar """ if cookiejar is None: cookiejar = RequestsCookieJar() @@ -529,6 +531,7 @@ def merge_cookies(cookiejar, cookies): :param cookiejar: CookieJar object to add the cookies to. :param cookies: Dictionary or CookieJar object to be added. + :rtype: CookieJar """ if not isinstance(cookiejar, cookielib.CookieJar): raise ValueError('You can only merge into CookieJar') diff --git a/pipenv/vendor/requests/help.py b/pipenv/vendor/requests/help.py index 06e06b2a..e53d35ef 100644 --- a/pipenv/vendor/requests/help.py +++ b/pipenv/vendor/requests/help.py @@ -89,8 +89,7 @@ def info(): 'version': getattr(idna, '__version__', ''), } - # OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module. - system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None) + system_ssl = ssl.OPENSSL_VERSION_NUMBER system_ssl_info = { 'version': '%x' % system_ssl if system_ssl is not None else '' } diff --git a/pipenv/vendor/requests/hooks.py b/pipenv/vendor/requests/hooks.py index 32b32de7..7a51f212 100644 --- a/pipenv/vendor/requests/hooks.py +++ b/pipenv/vendor/requests/hooks.py @@ -15,14 +15,14 @@ HOOKS = ['response'] def default_hooks(): - return dict((event, []) for event in HOOKS) + return {event: [] for event in HOOKS} # TODO: response is the only one def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" - hooks = hooks or dict() + hooks = hooks or {} hooks = hooks.get(key) if hooks: if hasattr(hooks, '__call__'): diff --git a/pipenv/vendor/requests/models.py b/pipenv/vendor/requests/models.py index 3d0e1f42..3dded57e 100644 --- a/pipenv/vendor/requests/models.py +++ b/pipenv/vendor/requests/models.py @@ -204,9 +204,13 @@ class Request(RequestHooksMixin): :param url: URL to send. :param headers: dictionary of headers to send. :param files: dictionary of {filename: fileobject} files to multipart upload. - :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. + :param data: the body to attach to the request. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param json: json for the body to attach to the request (if files or data is not specified). - :param params: dictionary of URL parameters to append to the URL. + :param params: URL parameters to append to the URL. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param auth: Auth handler or (user, pass) tuple. :param cookies: dictionary or CookieJar of cookies to attach to this request. :param hooks: dictionary of callback hooks, for internal usage. @@ -214,7 +218,7 @@ class Request(RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> req.prepare() """ @@ -274,7 +278,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> r = req.prepare() @@ -648,10 +652,7 @@ class Response(object): if not self._content_consumed: self.content - return dict( - (attr, getattr(self, attr, None)) - for attr in self.__attrs__ - ) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): diff --git a/pipenv/vendor/requests/sessions.py b/pipenv/vendor/requests/sessions.py index ba135268..a448bd83 100644 --- a/pipenv/vendor/requests/sessions.py +++ b/pipenv/vendor/requests/sessions.py @@ -115,6 +115,22 @@ class SessionRedirectMixin(object): return to_native_string(location, 'utf8') return None + def should_strip_auth(self, old_url, new_url): + """Decide whether Authorization header should be removed when redirecting""" + old_parsed = urlparse(old_url) + new_parsed = urlparse(new_url) + if old_parsed.hostname != new_parsed.hostname: + return True + # Special case: allow http -> https redirect when using the standard + # ports. This isn't specified by RFC 7235, but is kept to avoid + # breaking backwards compatibility with older versions of requests + # that allowed any redirects on the same host. + if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) + and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): + return False + # Standard case: root URI must match + return old_parsed.port != new_parsed.port or old_parsed.scheme != new_parsed.scheme + def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): """Receives a Response. Returns a generator of Responses or Requests.""" @@ -236,14 +252,10 @@ class SessionRedirectMixin(object): headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: + if 'Authorization' in headers and self.should_strip_auth(response.request.url, url): # If we get redirected to a new host, we should strip out any # authentication headers. - original_parsed = urlparse(response.request.url) - redirect_parsed = urlparse(url) - - if (original_parsed.hostname != redirect_parsed.hostname): - del headers['Authorization'] + del headers['Authorization'] # .netrc might have more auth for us on our new host. new_auth = get_netrc_auth(url) if self.trust_env else None @@ -299,7 +311,7 @@ class SessionRedirectMixin(object): """ method = prepared_request.method - # http://tools.ietf.org/html/rfc7231#section-6.4.4 + # https://tools.ietf.org/html/rfc7231#section-6.4.4 if response.status_code == codes.see_other and method != 'HEAD': method = 'GET' @@ -325,13 +337,13 @@ class Session(SessionRedirectMixin): >>> import requests >>> s = requests.Session() - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') Or as a context manager:: >>> with requests.Session() as s: - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') """ @@ -453,8 +465,8 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary, bytes, or file-like object to send - in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the @@ -550,7 +562,8 @@ class Session(SessionRedirectMixin): r"""Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response @@ -562,7 +575,8 @@ class Session(SessionRedirectMixin): r"""Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -573,7 +587,8 @@ class Session(SessionRedirectMixin): r"""Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -723,7 +738,7 @@ class Session(SessionRedirectMixin): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state def __setstate__(self, state): @@ -735,7 +750,12 @@ def session(): """ Returns a :class:`Session` for context-management. + .. deprecated:: 1.0.0 + + This method has been deprecated since version 1.0.0 and is only kept for + backwards compatibility. New code should use :class:`~requests.sessions.Session` + to create a session. This may be removed at a future date. + :rtype: Session """ - return Session() diff --git a/pipenv/vendor/requests/status_codes.py b/pipenv/vendor/requests/status_codes.py index ff462c6c..813e8c4e 100644 --- a/pipenv/vendor/requests/status_codes.py +++ b/pipenv/vendor/requests/status_codes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" +r""" The ``codes`` object defines a mapping from common names for HTTP statuses to their numerical codes, accessible either as attributes or as dictionary items. diff --git a/pipenv/vendor/requests/utils.py b/pipenv/vendor/requests/utils.py index 431f6be0..0ce7fe11 100644 --- a/pipenv/vendor/requests/utils.py +++ b/pipenv/vendor/requests/utils.py @@ -173,10 +173,10 @@ def get_netrc_auth(url, raise_errors=False): for f in NETRC_FILES: try: - loc = os.path.expanduser('~/{0}'.format(f)) + loc = os.path.expanduser('~/{}'.format(f)) except KeyError: # os.path.expanduser can fail when $HOME is undefined and - # getpwuid fails. See http://bugs.python.org/issue20164 & + # getpwuid fails. See https://bugs.python.org/issue20164 & # https://github.com/requests/requests/issues/1846 return @@ -466,7 +466,7 @@ def _parse_content_type_header(header): if index_of_equals != -1: key = param[:index_of_equals].strip(items_to_strip) value = param[index_of_equals + 1:].strip(items_to_strip) - params_dict[key] = value + params_dict[key.lower()] = value return content_type, params_dict @@ -706,6 +706,10 @@ def should_bypass_proxies(url, no_proxy): no_proxy = get_proxy('no_proxy') parsed = urlparse(url) + if parsed.hostname is None: + # URLs don't always have hostnames, e.g. file:/// urls. + return True + if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the hostname, both with and without the port. @@ -725,7 +729,7 @@ def should_bypass_proxies(url, no_proxy): else: host_with_port = parsed.hostname if parsed.port: - host_with_port += ':{0}'.format(parsed.port) + host_with_port += ':{}'.format(parsed.port) for host in no_proxy: if parsed.hostname.endswith(host) or host_with_port.endswith(host): @@ -733,13 +737,8 @@ def should_bypass_proxies(url, no_proxy): # to apply the proxies on this URL. return True - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - # The proxy_bypass function is incredibly buggy on OS X in early versions - # of Python 2.6, so allow this call to fail. Only catch the specific - # exceptions we've seen, though: this call failing in other ways can reveal - # legitimate problems. with set_environ('no_proxy', no_proxy_arg): + # parsed.hostname can be `None` in cases such as a file URI. try: bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index ae643517..d9f1b653 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -17,9 +17,10 @@ from pip_shims import ( FormatControl, InstallRequirement, PackageFinder, RequirementPreparer, RequirementSet, RequirementTracker, Resolver, WheelCache, pip_version ) -from vistir.compat import JSONDecodeError, TemporaryDirectory, fs_str +from vistir.compat import JSONDecodeError, fs_str from vistir.contextmanagers import cd, temp_environ from vistir.misc import partialclass +from vistir.path import create_tracked_tempdir from ..utils import get_pip_command, prepare_pip_source_args, _ensure_dir from .cache import CACHE_DIR, DependencyCache @@ -580,12 +581,12 @@ def start_resolver(finder=None, wheel_cache=None): download_dir = PKGS_DOWNLOAD_DIR _ensure_dir(download_dir) - _build_dir = TemporaryDirectory(fs_str("build")) - _source_dir = TemporaryDirectory(fs_str("source")) + _build_dir = create_tracked_tempdir(fs_str("build")) + _source_dir = create_tracked_tempdir(fs_str("source")) preparer = partialclass( RequirementPreparer, - build_dir=_build_dir.name, - src_dir=_source_dir.name, + build_dir=_build_dir, + src_dir=_source_dir, download_dir=download_dir, wheel_download_dir=WHEEL_DOWNLOAD_DIR, progress_bar="off", diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index bd76ca01..1997fc1f 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -14,6 +14,7 @@ from .project import ProjectFile from .requirements import Requirement from .utils import optional_instance_of +from ..utils import is_vcs, is_editable, merge_items DEFAULT_NEWLINES = u"\n" @@ -49,6 +50,27 @@ class Lockfile(object): def _get_lockfile(self): return self.projectfile.lockfile + def __getitem__(self, k, *args, **kwargs): + retval = None + lockfile = self._lockfile + section = None + pkg_type = None + try: + retval = lockfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(lockfile.get(section, {}), "_data", {}) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + def __getattr__(self, k, *args, **kwargs): retval = None lockfile = super(Lockfile, self).__getattribute__("_lockfile") @@ -56,9 +78,18 @@ class Lockfile(object): return super(Lockfile, self).__getattribute__(k) except AttributeError: retval = getattr(lockfile, k, None) - if not retval: - retval = super(Lockfile, self).__getattribute__(k, *args, **kwargs) - return retval + if retval is not None: + return retval + return super(Lockfile, self).__getattribute__(k, *args, **kwargs) + + def get_deps(self, dev=False, only=True): + deps = {} + if dev: + deps.update(self.develop._data) + if only: + return deps + deps = merge_items([deps, self.default._data]) + return deps @classmethod def read_projectfile(cls, path): @@ -135,7 +166,7 @@ class Lockfile(object): def default(self): return self._lockfile.default - def get_requirements(self, dev=False): + def get_requirements(self, dev=True, only=False): """Produces a generator which generates requirements from the desired section. :param bool dev: Indicates whether to use dev requirements, defaults to False @@ -143,20 +174,20 @@ class Lockfile(object): :rtype: :class:`~requirementslib.models.requirements.Requirement` """ - section = self.develop if dev else self.default - for k in section.keys(): - yield Requirement.from_pipfile(k, section[k]._data) + deps = self.get_deps(dev=dev, only=only) + for k, v in deps.items(): + yield Requirement.from_pipfile(k, v) @property def dev_requirements(self): if not self._dev_requirements: - self._dev_requirements = list(self.get_requirements(dev=True)) + self._dev_requirements = list(self.get_requirements(dev=True, only=True)) return self._dev_requirements @property def requirements(self): if not self._requirements: - self._requirements = list(self.get_requirements(dev=False)) + self._requirements = list(self.get_requirements(dev=False, only=True)) return self._requirements @property diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 94e9a2a1..0d1c04c8 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -6,12 +6,15 @@ import attr import copy import os +import tomlkit + from vistir.compat import Path, FileNotFoundError from .requirements import Requirement from .project import ProjectFile from .utils import optional_instance_of from ..exceptions import RequirementError +from ..utils import is_vcs, is_editable, merge_items import plette.pipfiles @@ -20,6 +23,31 @@ is_path = optional_instance_of(Path) is_projectfile = optional_instance_of(ProjectFile) +class PipfileLoader(plette.pipfiles.Pipfile): + @classmethod + def validate(cls, data): + for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items(): + if key not in data or key == "source": + continue + klass.validate(data[key]) + + @classmethod + def load(cls, f, encoding=None): + content = f.read() + if encoding is not None: + content = content.decode(encoding) + _data = tomlkit.loads(content) + if "source" not in _data: + # HACK: There is no good way to prepend a section to an existing + # TOML document, but there's no good way to copy non-structural + # content from one TOML document to another either. Modify the + # TOML content directly, and load the new in-memory document. + sep = "" if content.startswith("\n") else "\n" + content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content + data = tomlkit.loads(content) + return cls(data) + + @attr.s(slots=True) class Pipfile(object): path = attr.ib(validator=is_path, type=Path) @@ -40,16 +68,50 @@ class Pipfile(object): def _get_pipfile(self): return self.projectfile.model + @property + def pipfile(self): + return self._pipfile + + def get_deps(self, dev=False, only=True): + deps = {} + if dev: + deps.update(self.pipfile["dev-packages"]._data) + if only: + return deps + deps = merge_items([deps, self.pipfile["packages"]._data]) + return deps + + def __getitem__(self, k, *args, **kwargs): + retval = None + pipfile = self._pipfile + section = None + pkg_type = None + try: + retval = pipfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(pipfile.get(section, {}), "_data", {}) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + def __getattr__(self, k, *args, **kwargs): retval = None pipfile = super(Pipfile, self).__getattribute__("_pipfile") try: - return super(Pipfile, self).__getattribute__(k) + retval = super(Pipfile, self).__getattribute__(k) except AttributeError: retval = getattr(pipfile, k, None) - if not retval: - retval = super(Pipfile, self).__getattribute__(k, *args, **kwargs) - return retval + if retval is not None: + return retval + return super(Pipfile, self).__getattribute__(k, *args, **kwargs) @property def requires_python(self): @@ -69,7 +131,7 @@ class Pipfile(object): """ pf = ProjectFile.read( path, - plette.pipfiles.Pipfile, + PipfileLoader, invalid_ok=True ) return pf @@ -88,7 +150,7 @@ class Pipfile(object): if not path: raise RuntimeError("Must pass a path to classmethod 'Pipfile.load'") if not isinstance(path, Path): - path = Path(path) + path = Path(path).absolute() pipfile_path = path if path.name == "Pipfile" else path.joinpath("Pipfile") project_path = pipfile_path.parent if not project_path.exists(): @@ -113,10 +175,10 @@ class Pipfile(object): projectfile = cls.load_projectfile(path, create=create) pipfile = projectfile.model dev_requirements = [ - Requirement.from_pipfile(k, v._data) for k, v in pipfile.get("dev-packages", {}).items() + Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("dev-packages", {}).items() ] requirements = [ - Requirement.from_pipfile(k, v._data) for k, v in pipfile.get("packages", {}).items() + Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("packages", {}).items() ] creation_args = { "projectfile": projectfile, diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index c2768417..db004869 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -22,11 +22,11 @@ from pip_shims.shims import ( ) from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote -from vistir.compat import FileNotFoundError, Path, TemporaryDirectory +from vistir.compat import FileNotFoundError, Path from vistir.misc import dedup from vistir.path import ( create_tracked_tempdir, get_converted_relative_path, is_file_url, - is_valid_url, mkdir_p + is_valid_url ) from ..exceptions import RequirementError diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index b490d3cf..abc89831 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -5,9 +5,15 @@ import contextlib import logging import os +import boltons.iterutils import six import tomlkit +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) +from six.moves import Mapping, Sequence, Set, ItemsView from six.moves.urllib.parse import urlparse, urlsplit from pip_shims.shims import ( @@ -18,6 +24,8 @@ from vistir.compat import Path from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir + + VCS_LIST = ("git", "svn", "hg", "bzr") VCS_SCHEMES = [] SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") @@ -66,6 +74,12 @@ def is_vcs(pipfile_entry): return False +def is_editable(pipfile_entry): + if isinstance(pipfile_entry, Mapping): + return pipfile_entry.get("editable", False) is True + return False + + def multi_split(s, split): """Splits on multiple given separators.""" for r in split: @@ -181,3 +195,95 @@ def ensure_setup_py(base_dir): finally: if is_new: setup_py.unlink() + + +# Modified from https://github.com/mahmoud/boltons/blob/master/boltons/iterutils.py +def dict_path_enter(path, key, value): + if isinstance(value, six.string_types): + return value, False + elif isinstance(value, (Mapping, dict)): + return value.__class__(), ItemsView(value) + elif isinstance(value, tomlkit.items.Array): + return value.__class__([], value.trivia), enumerate(value) + elif isinstance(value, (Sequence, list)): + return value.__class__(), enumerate(value) + elif isinstance(value, (Set, set)): + return value.__class__(), enumerate(value) + else: + return value, False + + +def dict_path_exit(path, key, old_parent, new_parent, new_items): + ret = new_parent + if isinstance(new_parent, (Mapping, dict)): + vals = dict(new_items) + try: + new_parent.update(new_items) + except AttributeError: + # Handle toml containers specifically + try: + new_parent.update(vals) + # Now use default fallback if needed + except AttributeError: + ret = new_parent.__class__(vals) + elif isinstance(new_parent, tomlkit.items.Array): + vals = tomlkit.items.item([v for i, v in new_items]) + try: + new_parent._value.extend(vals._value) + except AttributeError: + ret = tomlkit.items.item(vals) + elif isinstance(new_parent, (Sequence, list)): + vals = [v for i, v in new_items] + try: + new_parent.extend(vals) + except AttributeError: + ret = new_parent.__class__(vals) # tuples + elif isinstance(new_parent, (Set, set)): + vals = [v for i, v in new_items] + try: + new_parent.update(vals) + except AttributeError: + ret = new_parent.__class__(vals) # frozensets + else: + raise RuntimeError('unexpected iterable type: %r' % type(new_parent)) + return ret + + +def merge_items(target_list, sourced=False): + if not sourced: + target_list = [(id(t), t) for t in target_list] + + ret = None + source_map = {} + + def remerge_enter(path, key, value): + new_parent, new_items = dict_path_enter(path, key, value) + if ret and not path and key is None: + new_parent = ret + + try: + cur_val = boltons.iterutils.get_path(ret, path + (key,)) + except KeyError as ke: + pass + else: + new_parent = cur_val + + return new_parent, new_items + + def remerge_exit(path, key, old_parent, new_parent, new_items): + return dict_path_exit(path, key, old_parent, new_parent, new_items) + + for t_name, target in target_list: + if sourced: + def remerge_visit(path, key, value): + source_map[path + (key,)] = t_name + return True + else: + remerge_visit = boltons.iterutils.default_visit + + ret = boltons.iterutils.remap(target, enter=remerge_enter, visit=remerge_visit, + exit=remerge_exit) + + if not sourced: + return ret + return ret, source_map diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 70f95c59..8f25e079 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -129,7 +129,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): ) else: spinner_name = None - if not start_text: + if not start_text and nospin is False: start_text = "Running..." with spinner_func( spinner_name=spinner_name, diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index f42b4ad1..e2f85985 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -10,6 +10,7 @@ import sys from collections import OrderedDict from functools import partial +from itertools import islice import six @@ -32,6 +33,9 @@ __all__ = [ "to_text", "to_bytes", "locale_encoding", + "chunked", + "take", + "divide" ] @@ -283,7 +287,10 @@ def run( cmd = Script.parse(cmd) if block or not return_object: combine_stderr = False - with spinner(spinner_name=spinner_name, start_text="Running...", nospin=nospin) as sp: + start_text = "Running..." + if nospin: + start_text = None + with spinner(spinner_name=spinner_name, start_text=start_text, nospin=nospin) as sp: return _create_subprocess( cmd, env=_env, @@ -427,6 +434,52 @@ def to_text(string, encoding="utf-8", errors=None): return string +def divide(n, iterable): + """ + split an iterable into n groups, per https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + :return: a list of new iterables derived from the original iterable + :rtype: list + """ + + seq = tuple(iterable) + q, r = divmod(len(seq), n) + + ret = [] + for i in range(n): + start = (i * q) + (i if i < r else r) + stop = ((i + 1) * q) + (i + 1 if i + 1 < r else r) + ret.append(iter(seq[start:stop])) + + return ret + + +def take(n, iterable): + """Take n elements from the supplied iterable without consuming it. + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + + from https://github.com/erikrose/more-itertools/blob/master/more_itertools/recipes.py + """ + + return list(islice(iterable, n)) + + +def chunked(n, iterable): + """Split an iterable into lists of length *n*. + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + + from https://github.com/erikrose/more-itertools/blob/master/more_itertools/more.py + """ + + return iter(partial(take, n, iter(iterable)), []) + + try: locale_encoding = locale.getdefaultencoding()[1] or "ascii" except Exception: From e375eb3eaa9734f8124c5f7db8aeae3301629eff Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 22 Oct 2018 10:13:34 -0400 Subject: [PATCH 013/218] Fix delegator leaky file handles Signed-off-by: Dan Ryan --- pipenv/vendor/delegator.py | 21 ++++++--- .../vendor/delegator-close-filehandles.patch | 44 +++++++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 tasks/vendoring/patches/vendor/delegator-close-filehandles.patch diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py index d15aeb97..3ffb2e31 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py @@ -178,6 +178,7 @@ class Command(object): # Use subprocess. if self.blocking: popen_kwargs = self._default_popen_kwargs.copy() + del popen_kwargs["stdin"] popen_kwargs["universal_newlines"] = not binary if cwd: popen_kwargs["cwd"] = cwd @@ -234,14 +235,22 @@ class Command(object): """Blocks until process is complete.""" if self._uses_subprocess: # consume stdout and stderr - try: - stdout, stderr = self.subprocess.communicate() - self.__out = stdout - self.__err = stderr - except ValueError: - pass # Don't read from finished subprocesses. + if self.blocking: + try: + stdout, stderr = self.subprocess.communicate() + self.__out = stdout + self.__err = stderr + except ValueError: + pass # Don't read from finished subprocesses. + else: + self.subprocess.stdin.close() + self.std_out.close() + self.std_err.close() + self.subprocess.wait() else: + self.subprocess.sendeof() self.subprocess.wait() + self.subprocess.proc.stdout.close() def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch new file mode 100644 index 00000000..ac63825c --- /dev/null +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -0,0 +1,44 @@ +diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py +index 0c140cad..3ffb2e31 100644 +--- a/pipenv/vendor/delegator.py ++++ b/pipenv/vendor/delegator.py +@@ -178,6 +178,7 @@ class Command(object): + # Use subprocess. + if self.blocking: + popen_kwargs = self._default_popen_kwargs.copy() ++ del popen_kwargs["stdin"] + popen_kwargs["universal_newlines"] = not binary + if cwd: + popen_kwargs["cwd"] = cwd +@@ -233,18 +234,23 @@ class Command(object): + def block(self): + """Blocks until process is complete.""" + if self._uses_subprocess: +- self.subprocess.stdin.close() + # consume stdout and stderr +- try: +- stdout, stderr = self.subprocess.communicate() +- self.__out = stdout +- self.__err = stderr +- except ValueError: +- pass # Don't read from finished subprocesses. ++ if self.blocking: ++ try: ++ stdout, stderr = self.subprocess.communicate() ++ self.__out = stdout ++ self.__err = stderr ++ except ValueError: ++ pass # Don't read from finished subprocesses. ++ else: ++ self.subprocess.stdin.close() ++ self.std_out.close() ++ self.std_err.close() ++ self.subprocess.wait() + else: + self.subprocess.sendeof() +- self.subprocess.proc.stdout.close() + self.subprocess.wait() ++ self.subprocess.proc.stdout.close() + + def pipe(self, command, timeout=None, cwd=None): + """Runs the current command and passes its output to the next From 4dac1676579473e257f68686abdc94191dee237d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 22 Oct 2018 10:16:04 -0400 Subject: [PATCH 014/218] Vendor boltons Signed-off-by: Dan Ryan Update vendored dependencies Signed-off-by: Dan Ryan Fix file handle leaks - Fix #3020 - Fix #3088 - Patch delegator - Add weakref finalizer for tempfiles Signed-off-by: Dan Ryan Fix spinner handlers on windows Signed-off-by: Dan Ryan Fix spinner output and encoding issue Signed-off-by: Dan Ryan fix encoding Signed-off-by: Dan Ryan Fix unicode output on windows, fix tomlkit imports Signed-off-by: Dan Ryan Unvendor boltons, fix compatibility, update merge functionalities Signed-off-by: Dan Ryan Update pythonfinder, vistir version, requirementslib version Signed-off-by: Dan Ryan Fix vendoring script Signed-off-by: Dan Ryan Silence pip version checks Signed-off-by: Dan Ryan Add debugging to locking Signed-off-by: Dan Ryan --- news/{3020.feature => 3020.feature.rst} | 0 news/3088.bugfix.rst | 1 + news/3089.feature.rst | 1 + news/3089.vendor.rst | 11 + pipenv/__init__.py | 1 + pipenv/_compat.py | 6 +- pipenv/core.py | 535 +++++++++--------- pipenv/environments.py | 9 +- pipenv/project.py | 76 ++- pipenv/resolver.py | 86 +-- pipenv/utils.py | 195 +++---- pipenv/vendor/delegator.py | 17 +- pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/models/path.py | 1 - pipenv/vendor/requirementslib/__init__.py | 9 +- pipenv/vendor/requirementslib/exceptions.py | 2 + pipenv/vendor/requirementslib/models/cache.py | 16 +- .../vendor/requirementslib/models/lockfile.py | 26 + .../vendor/requirementslib/models/pipfile.py | 14 +- .../requirementslib/models/requirements.py | 48 +- .../requirementslib/models/resolvers.py | 11 +- pipenv/vendor/requirementslib/models/utils.py | 23 +- pipenv/vendor/requirementslib/utils.py | 283 ++++++++- pipenv/vendor/tomlkit/items.py | 2 +- pipenv/vendor/tomlkit/toml_char.py | 2 +- pipenv/vendor/vendor.txt | 6 +- pipenv/vendor/vistir/__init__.py | 7 +- pipenv/vendor/vistir/backports/tempfile.py | 6 +- pipenv/vendor/vistir/compat.py | 3 +- pipenv/vendor/vistir/contextmanagers.py | 26 +- pipenv/vendor/vistir/misc.py | 6 +- pipenv/vendor/vistir/path.py | 71 ++- pipenv/vendor/vistir/spin.py | 49 +- pipenv/vendor/vistir/termcolors.py | 4 +- pipenv/vendor/yaspin/core.py | 27 +- tasks/vendoring/__init__.py | 2 + .../vendor/delegator-close-filehandles.patch | 57 +- .../patches/vendor/vistir-imports.patch | 49 +- .../vendor/yaspin-signal-handling.patch | 62 ++ tests/integration/conftest.py | 138 ++++- tests/integration/test_lock.py | 4 +- tests/unit/test_utils.py | 11 +- 42 files changed, 1339 insertions(+), 566 deletions(-) rename news/{3020.feature => 3020.feature.rst} (100%) create mode 100644 news/3088.bugfix.rst create mode 100644 news/3089.feature.rst create mode 100644 news/3089.vendor.rst create mode 100644 tasks/vendoring/patches/vendor/yaspin-signal-handling.patch diff --git a/news/3020.feature b/news/3020.feature.rst similarity index 100% rename from news/3020.feature rename to news/3020.feature.rst diff --git a/news/3088.bugfix.rst b/news/3088.bugfix.rst new file mode 100644 index 00000000..b10c4b2b --- /dev/null +++ b/news/3088.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which caused ``Unexpected EOF`` errors to be thrown when PIP awaited input from users who put login credentials in their environment. diff --git a/news/3089.feature.rst b/news/3089.feature.rst new file mode 100644 index 00000000..47f280ee --- /dev/null +++ b/news/3089.feature.rst @@ -0,0 +1 @@ +Added windows-compatible spinner via upgraded ``vistir`` dependency. diff --git a/news/3089.vendor.rst b/news/3089.vendor.rst new file mode 100644 index 00000000..c57209f6 --- /dev/null +++ b/news/3089.vendor.rst @@ -0,0 +1,11 @@ +Updated vendored dependencies: + - ``certifi 2018.08.24 => 2018.10.15`` + - ``urllib3 1.23 => 1.24`` + - ``requests 2.19.1 => 2.20.0`` + - ``shellingham ``1.2.6 => 1.2.7`` + - ``tomlkit 0.4.4. => 0.4.6`` + - ``vistir 0.1.6 => 0.1.8`` + - ``pythonfinder 0.1.2 => 0.1.3`` + - ``requirementslib 1.1.9 => 1.1.10`` + - ``backports.functools_lru_cache 1.5.0 (new)`` + - ``cursor 1.2.0 (new)`` diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 471dcb94..1fea44d5 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -14,6 +14,7 @@ PIPENV_PATCHED = os.sep.join([PIPENV_ROOT, "patched"]) sys.path.insert(0, PIPENV_VENDOR) # Inject patched directory into system path. sys.path.insert(0, PIPENV_PATCHED) +os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" # Hack to make things work better. try: if "concurrency" in sys.modules: diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 558df3b8..223baec1 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -59,10 +59,10 @@ except ImportError: return False -if six.PY2: +from vistir.compat import ResourceWarning - class ResourceWarning(Warning): - pass + +warnings.filterwarnings("ignore", category=ResourceWarning) def pip_import(module_path, subimport=None, old_path=None): diff --git a/pipenv/core.py b/pipenv/core.py index ca37bd5c..82c11f1d 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -5,7 +5,6 @@ import os import sys import shutil import time -import tempfile import json as simplejson import click import click_completion @@ -13,10 +12,12 @@ import crayons import dotenv import delegator import pipfile -from blindspin import spinner import vistir +import warnings import six +import urllib3.util as urllib3_util + from .cmdparse import Script from .project import Project, SourceNotFound from .utils import ( @@ -24,8 +25,6 @@ from .utils import ( is_required_version, proper_case, pep423_name, - split_file, - merge_deps, venv_resolve_deps, escape_grouped_arguments, python_version, @@ -45,9 +44,7 @@ from .utils import ( from . import environments, pep508checker, progress from .environments import ( PIPENV_COLORBLIND, - PIPENV_NOSPIN, PIPENV_SHELL_FANCY, - PIPENV_TIMEOUT, PIPENV_SKIP_VALIDATION, PIPENV_HIDE_EMOJIS, PIPENV_YES, @@ -96,12 +93,36 @@ click_completion.init() # Disable colors, for the color blind and others who do not prefer colors. if PIPENV_COLORBLIND: crayons.disable() -# Disable spinner, for cleaner build logs (the unworthy). -if PIPENV_NOSPIN: - @contextlib.contextmanager # noqa: F811 - def spinner(): - yield + +UNICODE_TO_ASCII_TRANSLATION_MAP = { + 8230: u"...", + 8211: u"-" +} + + +def fix_utf8(text): + if not isinstance(text, six.string_types): + return text + if six.PY2: + text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) + return u"{0}".format(text) + + +@contextlib.contextmanager +def spinner(text=None, nospin=None, spinner_name=None): + if not text: + text = "Running..." + if not spinner_name: + spinner_name = environments.PIPENV_SPINNER + if nospin is None: + nospin = environments.PIPENV_NOSPIN + with vistir.spin.create_spinner( + spinner_name=spinner_name, + start_text=text, + nospin=nospin + ) as sp: + yield sp def which(command, location=None, allow_global=False): @@ -132,7 +153,7 @@ project = Project(which=which) def do_clear(): - click.echo(crayons.white("Clearing caches…", bold=True)) + click.echo(crayons.white(fix_utf8("Clearing caches…"), bold=True)) try: from pip._internal import locations except ImportError: # pip 9. @@ -161,7 +182,7 @@ def load_dot_env(): if os.path.isfile(dotenv_file): click.echo( - crayons.normal("Loading .env environment variables…", bold=True), + crayons.normal(fix_utf8("Loading .env environment variables…"), bold=True), err=True, ) else: @@ -295,7 +316,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): if project.requirements_exists and not skip_requirements: click.echo( crayons.normal( - u"requirements.txt found, instead of Pipfile! Converting…", + fix_utf8("requirements.txt found, instead of Pipfile! Converting…"), bold=True, ) ) @@ -317,7 +338,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): ) else: click.echo( - crayons.normal(u"Creating a Pipfile for this project…", bold=True), + crayons.normal(fix_utf8("Creating a Pipfile for this project…"), bold=True), err=True, ) # Create the pipfile if it doesn't exist. @@ -405,7 +426,7 @@ def ensure_python(three=None, python=None): u"{0}: Python {1} {2}".format( crayons.red("Warning", bold=True), crayons.blue(python), - u"was not found on your system…", + fix_utf8("was not found on your system…"), ), err=True, ) @@ -424,7 +445,7 @@ def ensure_python(three=None, python=None): except ValueError: abort() except PyenvError as e: - click.echo(u"Something went wrong…") + click.echo(fix_utf8("Something went wrong…")) click.echo(crayons.blue(e.err), err=True) abort() s = "{0} {1} {2}".format( @@ -443,14 +464,14 @@ def ensure_python(three=None, python=None): crayons.green(u"CPython {0}".format(version), bold=True), crayons.normal(u"with pyenv", bold=True), crayons.normal(u"(this may take a few minutes)"), - crayons.normal(u"…", bold=True), + crayons.normal(fix_utf8("…"), bold=True), ) ) with spinner(): try: c = pyenv.install(version) except PyenvError as e: - click.echo(u"Something went wrong…") + click.echo(fix_utf8("Something went wrong…")) click.echo(crayons.blue(e.err), err=True) # Print the results, in a beautiful blue… click.echo(crayons.blue(c.out), err=True) @@ -518,7 +539,7 @@ def ensure_virtualenv(three=None, python=None, site_packages=False, pypi_mirror= ): abort() click.echo( - crayons.normal(u"Removing existing virtualenv…", bold=True), err=True + crayons.normal(fix_utf8("Removing existing virtualenv…"), bold=True), err=True ) # Remove the virtualenv. cleanup_virtualenv(bare=True) @@ -665,26 +686,31 @@ def do_install_dependencies( If requirements is True, simply spits out a requirements format to stdout. """ - from .vendor.requirementslib.models.requirements import Requirement + from six.moves import queue def cleanup_procs(procs, concurrent): - for c in procs: - if concurrent: - c.block() + while not procs.empty(): + c = procs.get() + # if concurrent: + c.block() + failed = False + if c.return_code != 0: + failed = True if "Ignoring" in c.out: click.echo(crayons.yellow(c.out.strip())) elif environments.is_verbose(): - click.echo(crayons.blue(c.out or c.err)) + click.echo(crayons.blue(c.out.strip() or c.err.strip())) # The Installation failed… - if c.return_code != 0: + if failed: # Save the Failed Dependency for later. - failed_deps_list.append((c.dep, c.ignore_hash)) + dep = c.dep.copy() + failed_deps_list.append(dep) # Alert the user. click.echo( "{0} {1}! Will try again.".format( crayons.red("An error occurred while installing"), - crayons.green(c.dep.as_line()), - ) + crayons.green(dep.as_line()), + ), err=True ) if requirements: @@ -694,143 +720,138 @@ def do_install_dependencies( if skip_lock or only or not project.lockfile_exists: if not bare: click.echo( - crayons.normal(u"Installing dependencies from Pipfile…", bold=True) + crayons.normal(fix_utf8("Installing dependencies from Pipfile…"), bold=True) ) - lockfile = split_file(project._lockfile) + lockfile = project.get_or_create_lockfile() else: - with open(project.lockfile_location) as f: - lockfile = split_file(simplejson.load(f)) + lockfile = project.get_or_create_lockfile() if not bare: click.echo( crayons.normal( - u"Installing dependencies from Pipfile.lock ({0})…".format( + fix_utf8("Installing dependencies from Pipfile.lock ({0})…".format( lockfile["_meta"].get("hash", {}).get("sha256")[-6:] - ), + )), bold=True, ) ) # Allow pip to resolve dependencies when in skip-lock mode. no_deps = not skip_lock - deps_list, dev_deps_list = merge_deps( - lockfile, - project, - dev=dev, - requirements=requirements, - ignore_hashes=ignore_hashes, - blocking=blocking, - only=only, - ) failed_deps_list = [] + deps_list = list(lockfile.get_requirements(dev=dev, only=only)) if requirements: index_args = prepare_pip_source_args(project.sources) index_args = " ".join(index_args).replace(" -", "\n-") - deps_list = [dep for dep, ignore_hash, block in deps_list] - dev_deps_list = [dep for dep, ignore_hash, block in dev_deps_list] + deps = [ + req.as_line(sources=project.sources, include_hashes=False) for req in deps_list + ] # Output only default dependencies click.echo(index_args) - if not dev: - click.echo( - "\n".join(d.partition("--hash")[0].strip() for d in sorted(deps_list)) - ) - sys.exit(0) - # Output only dev dependencies - if dev: - click.echo( - "\n".join( - d.partition("--hash")[0].strip() for d in sorted(dev_deps_list) - ) - ) - sys.exit(0) - procs = [] - deps_list_bar = progress.bar( - deps_list, label=INSTALL_LABEL if os.name != "nt" else "" - ) - for dep, ignore_hash, block in deps_list_bar: - if len(procs) < PIPENV_MAX_SUBPROCESS: - # Use a specific index, if specified. - indexes, trusted_hosts, dep = parse_indexes(dep) - index = None - extra_indexes = [] - if indexes: - index = indexes[0] - if len(indexes) > 0: - extra_indexes = indexes[1:] - dep = Requirement.from_line(" ".join(dep)) - if index: - _index = None - try: - _index = project.find_source(index).get("name") - except SourceNotFound: - _index = None - dep.index = _index - dep._index = index - dep.extra_indexes = extra_indexes - # Install the module. - prev_no_deps_setting = no_deps - if dep.is_file_or_url and any( - dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] - ): - no_deps = False + click.echo( + "\n".join(sorted(deps)) + ) + sys.exit(0) + + procs = queue.Queue(maxsize=PIPENV_MAX_SUBPROCESS) + trusted_hosts = [] + + deps_list_bar = progress.bar(deps_list, width=32, + label=INSTALL_LABEL if os.name != "nt" else "") + + indexes = [] + for dep in deps_list_bar: + index = None + if dep.index: + index = project.find_source(dep.index) + indexes.append(index) + if not index.get("verify_ssl", False): + trusted_hosts.append(urllib3_util.parse_url(index.get("url")).host) + # Install the module. + is_artifact = False + if dep.is_file_or_url and any( + dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] + ): + is_artifact = True + + extra_indexes = [] + if not index and indexes: + index = next(iter(indexes)) + if len(indexes) > 1: + extra_indexes = indexes[1:] + with vistir.contextmanagers.temp_environ(): + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] c = pip_install( dep, - ignore_hashes=ignore_hash, + ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), allow_global=allow_global, - no_deps=no_deps, - block=block, + no_deps=False if is_artifact else no_deps, + block=any([dep.editable, blocking]), index=index, requirements_dir=requirements_dir, - extra_indexes=extra_indexes, pypi_mirror=pypi_mirror, - trusted_hosts=trusted_hosts + trusted_hosts=trusted_hosts, + extra_indexes=extra_indexes ) - c.dep = dep - c.ignore_hash = ignore_hash - c.index = index - c.extra_indexes = extra_indexes - procs.append(c) - no_deps = prev_no_deps_setting - if len(procs) >= PIPENV_MAX_SUBPROCESS or len(procs) == len(deps_list): - cleanup_procs(procs, concurrent) - procs = [] - cleanup_procs(procs, concurrent) + if procs.qsize() < PIPENV_MAX_SUBPROCESS: + c.dep = dep + procs.put(c) + + if procs.full() or procs.qsize() == len(deps_list): + cleanup_procs(procs, concurrent) + if not procs.empty(): + cleanup_procs(procs, concurrent) + # Iterate over the hopefully-poorly-packaged dependencies… if failed_deps_list: click.echo( - crayons.normal(u"Installing initially failed dependencies…", bold=True) + crayons.normal(fix_utf8("Installing initially failed dependencies…"), bold=True) ) - for dep, ignore_hash in progress.bar(failed_deps_list, label=INSTALL_LABEL2): + for dep in progress.bar(failed_deps_list, label=INSTALL_LABEL2): # Use a specific index, if specified. # Install the module. - prev_no_deps_setting = no_deps + is_artifact = False + index = None + if dep.index: + index = project.find_source(dep.index) if dep.is_file_or_url and any( dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] ): - no_deps = False - c = pip_install( - dep, - ignore_hashes=ignore_hash, - allow_global=allow_global, - no_deps=no_deps, - index=getattr(dep, "_index", None), - requirements_dir=requirements_dir, - extra_indexes=getattr(dep, "extra_indexes", None), - ) - no_deps = prev_no_deps_setting - # The Installation failed… - if c.return_code != 0: - # We echo both c.out and c.err because pip returns error details on out. - click.echo(crayons.blue(format_pip_output(c.out))) - click.echo(crayons.blue(format_pip_error(c.err)), err=True) - # Return the subprocess' return code. - sys.exit(c.return_code) - else: - click.echo( - "{0} {1}{2}".format( - crayons.green("Success installing"), - crayons.green(dep.name), - crayons.green("!"), - ) + is_artifact = True + extra_indexes = [] + if not index and indexes: + index = next(iter(indexes)) + if len(indexes) > 1: + extra_indexes = indexes[1:] + with vistir.contextmanagers.temp_environ(): + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] + c = pip_install( + dep, + ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), + allow_global=allow_global, + no_deps=True if is_artifact else no_deps, + index=index, + requirements_dir=requirements_dir, + pypi_mirror=pypi_mirror, + trusted_hosts=trusted_hosts, + extra_indexes=extra_indexes, + block=True ) + # The Installation failed… + if c.return_code != 0: + # We echo both c.out and c.err because pip returns error details on out. + click.echo(crayons.blue(format_pip_output(c.out))) + click.echo(crayons.blue(format_pip_error(c.err)), err=True) + # Return the subprocess' return code. + sys.exit(c.return_code) + else: + click.echo( + "{0} {1}{2}".format( + crayons.green("Success installing"), + crayons.green(dep.name), + crayons.green("!"), + ) + ) def convert_three_to_python(three, python): @@ -851,7 +872,7 @@ def convert_three_to_python(three, python): def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): """Creates a virtualenv.""" click.echo( - crayons.normal(u"Creating a virtualenv for this project…", bold=True), err=True + crayons.normal(fix_utf8("Creating a virtualenv for this project…"), bold=True), err=True ) click.echo( u"Pipfile: {0}".format(crayons.red(project.pipfile_location, bold=True)), @@ -865,14 +886,14 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): u"{0} {1} {3} {2}".format( crayons.normal("Using", bold=True), crayons.red(python, bold=True), - crayons.normal(u"to create virtualenv…", bold=True), + crayons.normal(fix_utf8("to create virtualenv…"), bold=True), crayons.green("({0})".format(python_version(python))), ), err=True, ) cmd = [ - sys.executable, + vistir.compat.Path(sys.executable).absolute().as_posix(), "-m", "virtualenv", "--prompt=({0}) ".format(project.name), @@ -883,7 +904,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): # Pass site-packages flag to virtualenv, if desired… if site_packages: click.echo( - crayons.normal(u"Making site-packages available…", bold=True), err=True + crayons.normal(fix_utf8("Making site-packages available…"), bold=True), err=True ) cmd.append("--system-site-packages") @@ -893,11 +914,12 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): pip_config = {} # Actually create the virtualenv. - with spinner(): - c = delegator.run(cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config) - c.block() + nospin = os.environ.get("PIPENV_ACTIVE", environments.PIPENV_NOSPIN) + c = vistir.misc.run(cmd, verbose=False, return_object=True, + spinner_name=environments.PIPENV_SPINNER, combine_stderr=False, + block=False, nospin=nospin, env=pip_config) click.echo(crayons.blue("{0}".format(c.out)), err=True) - if c.return_code != 0: + if c.returncode != 0: click.echo(crayons.blue("{0}".format(c.err)), err=True) click.echo( u"{0}: Failed to create virtual environment.".format( @@ -1019,9 +1041,9 @@ def do_lock( # Alert the user of progress. click.echo( u"{0} {1} {2}".format( - crayons.normal("Locking"), - crayons.red("[{0}]".format(settings["log_string"])), - crayons.normal("dependencies…"), + crayons.normal(u"Locking"), + crayons.red(u"[{0}]".format(settings["log_string"])), + crayons.normal(fix_utf8("dependencies…")), ), err=True, ) @@ -1125,7 +1147,7 @@ def do_purge(bare=False, downloads=False, allow_global=False): if downloads: if not bare: - click.echo(crayons.normal(u"Clearing out downloads directory…", bold=True)) + click.echo(crayons.normal(fix_utf8("Clearing out downloads directory…"), bold=True)) shutil.rmtree(project.download_location) return @@ -1154,7 +1176,7 @@ def do_purge(bare=False, downloads=False, allow_global=False): actually_installed.append(dep) if not bare: click.echo( - u"Found {0} installed package(s), purging…".format(len(actually_installed)) + fix_utf8("Found {0} installed package(s), purging…".format(len(actually_installed))) ) command = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=allow_global)), @@ -1185,7 +1207,6 @@ def do_init( """Executes the init functionality.""" from .environments import PIPENV_VIRTUALENV - cleanup_reqdir = False if not system: if not project.virtualenv_exists: try: @@ -1197,8 +1218,7 @@ def do_init( if not deploy: ensure_pipfile(system=system) if not requirements_dir: - cleanup_reqdir = True - requirements_dir = vistir.compat.TemporaryDirectory( + requirements_dir = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored @@ -1215,26 +1235,25 @@ def do_init( ) ) click.echo(crayons.normal("Aborting deploy.", bold=True), err=True) - requirements_dir.cleanup() sys.exit(1) elif (system or allow_global) and not (PIPENV_VIRTUALENV): click.echo( - crayons.red( - u"Pipfile.lock ({0}) out of date, but installation " - u"uses {1}… re-building lockfile must happen in " - u"isolation. Please rebuild lockfile in a virtualenv. " - u"Continuing anyway…".format( + crayons.red(fix_utf8( + "Pipfile.lock ({0}) out of date, but installation " + "uses {1}… re-building lockfile must happen in " + "isolation. Please rebuild lockfile in a virtualenv. " + "Continuing anyway…".format( crayons.white(old_hash[-6:]), crayons.white("--system") - ), + )), bold=True, ), err=True, ) else: if old_hash: - msg = u"Pipfile.lock ({1}) out of date, updating to ({0})…" + msg = fix_utf8("Pipfile.lock ({1}) out of date, updating to ({0})…") else: - msg = u"Pipfile.lock is corrupted, replaced with ({0})…" + msg = fix_utf8("Pipfile.lock is corrupted, replaced with ({0})…") click.echo( crayons.red(msg.format(old_hash[-6:], new_hash[-6:]), bold=True), err=True, @@ -1259,11 +1278,10 @@ def do_init( err=True, ) click.echo("See also: --deploy flag.", err=True) - requirements_dir.cleanup() sys.exit(1) else: click.echo( - crayons.normal(u"Pipfile.lock not found, creating…", bold=True), + crayons.normal(fix_utf8("Pipfile.lock not found, creating…"), bold=True), err=True, ) do_lock( @@ -1279,11 +1297,9 @@ def do_init( allow_global=allow_global, skip_lock=skip_lock, concurrent=concurrent, - requirements_dir=requirements_dir.name, + requirements_dir=requirements_dir, pypi_mirror=pypi_mirror, ) - if cleanup_reqdir: - requirements_dir.cleanup() # Hint the user what to do to activate the virtualenv. if not allow_global and not deploy and "PIPENV_ACTIVE" not in os.environ: @@ -1312,14 +1328,14 @@ def pip_install( trusted_hosts=None ): from notpip._internal import logger as piplogger + from .utils import Mapping from .vendor.urllib3.util import parse_url src = [] write_to_tmpfile = False if requirement: - editable_with_markers = requirement.editable and requirement.markers needs_hashes = not requirement.editable and not ignore_hashes and r is None - write_to_tmpfile = needs_hashes or editable_with_markers + write_to_tmpfile = needs_hashes if not trusted_hosts: trusted_hosts = [] @@ -1333,12 +1349,16 @@ def pip_install( ) # Create files for hash mode. if write_to_tmpfile: - with vistir.compat.NamedTemporaryFile( + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir( + prefix="pipenv", suffix="requirements") + f = vistir.compat.NamedTemporaryFile( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, delete=False - ) as f: - f.write(vistir.misc.to_bytes(requirement.as_line())) - r = f.name + ) + f.write(vistir.misc.to_bytes(requirement.as_line())) + r = f.name + f.close() # Install dependencies when a package is a VCS dependency. if requirement and requirement.vcs: no_deps = False @@ -1348,21 +1368,27 @@ def pip_install( # Try installing for each source in project.sources. if index: - try: - index_source = project.find_source(index) - index_source = index_source.copy() - except SourceNotFound: - src_name = project.src_name_from_url(index) - index_url = parse_url(index) - verify_ssl = index_url.host not in trusted_hosts - index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} + if isinstance(index, (Mapping, dict)): + index_source = index + else: + try: + index_source = project.find_source(index) + index_source = index_source.copy() + except SourceNotFound: + src_name = project.src_name_from_url(index) + index_url = parse_url(index) + verify_ssl = index_url.host not in trusted_hosts + index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} sources = [index_source.copy(),] if extra_indexes: if isinstance(extra_indexes, six.string_types): extra_indexes = [extra_indexes,] for idx in extra_indexes: + extra_src = None + if isinstance(idx, (Mapping, dict)): + extra_src = idx try: - extra_src = project.find_source(idx) + extra_src = project.find_source(idx) if not extra_src else extra_src except SourceNotFound: src_name = project.src_name_from_url(idx) src_url = parse_url(idx) @@ -1382,12 +1408,15 @@ def pip_install( for source in sources ] if (requirement and requirement.editable) and not r: - install_reqs = requirement.as_line(as_list=True) + line_kwargs = {"as_list": True} + if requirement.markers: + line_kwargs["include_markers"] = False + install_reqs = requirement.as_line(**line_kwargs) if requirement.editable and install_reqs[0].startswith("-e "): req, install_reqs = install_reqs[0], install_reqs[1:] editable_opt, req = req.split(" ", 1) install_reqs = [editable_opt, req] + install_reqs - if not any(item.startswith("--hash") for item in install_reqs): + if not all(item.startswith("--hash") for item in install_reqs): ignore_hashes = True elif r: install_reqs = ["-r", r] @@ -1396,7 +1425,7 @@ def pip_install( ignore_hashes = True else: ignore_hashes = True if not requirement.hashes else False - install_reqs = requirement.as_line(as_list=True) + install_reqs = [escape_cmd(r) for r in requirement.as_line(as_list=True)] pip_command = [which_pip(allow_global=allow_global), "install"] if pre: pip_command.append("--pre") @@ -1409,7 +1438,6 @@ def pip_install( pip_command.append("--upgrade-strategy=only-if-needed") if no_deps: pip_command.append("--no-deps") - install_reqs = [escape_cmd(req) for req in install_reqs] pip_command.extend(install_reqs) pip_command.extend(prepare_pip_source_args(sources)) if not ignore_hashes: @@ -1431,7 +1459,8 @@ def pip_install( pip_config.update( {"PIP_SRC": vistir.misc.fs_str(project.virtualenv_src_location)} ) - pip_command = Script.parse(pip_command).cmdify() + cmd = Script.parse(pip_command) + pip_command = cmd.cmdify() c = delegator.run(pip_command, block=block, env=pip_config) return c @@ -1623,9 +1652,9 @@ def ensure_lockfile(keep_outdated=False, pypi_mirror=None): if new_hash != old_hash: click.echo( crayons.red( - u"Pipfile.lock ({0}) out of date, updating to ({1})…".format( + fix_utf8("Pipfile.lock ({0}) out of date, updating to ({1})…".format( old_hash[-6:], new_hash[-6:] - ), + )), bold=True, ), err=True, @@ -1700,9 +1729,10 @@ def do_install( from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM from notpip._internal.exceptions import PipError - requirements_directory = vistir.compat.TemporaryDirectory( + requirements_directory = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) + warnings.filterwarnings("default", category=vistir.compat.ResourceWarning) if selective_upgrade: keep_outdated = True packages = packages if packages else [] @@ -1739,27 +1769,29 @@ def do_install( err=True, ) click.echo("See also: --deploy flag.", err=True) - requirements_directory.cleanup() sys.exit(1) # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True # Check if the file is remote or not if remote: - fd, temp_reqs = tempfile.mkstemp( - prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory.name - ) - requirements_url = requirements - # Download requirements file click.echo( crayons.normal( - u"Remote requirements file provided! Downloading…", bold=True + fix_utf8("Remote requirements file provided! Downloading…"), bold=True ), err=True, ) + fd = vistir.path.create_tracked_tempfile( + prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory + ) + temp_reqs = fd.name + requirements_url = requirements + # Download requirements file try: download_file(requirements, temp_reqs) except IOError: + fd.close() + os.unlink(temp_reqs) click.echo( crayons.red( u"Unable to find requirements file at {0}.".format( @@ -1768,8 +1800,9 @@ def do_install( ), err=True, ) - requirements_directory.cleanup() sys.exit(1) + finally: + fd.close() # Replace the url with the temporary requirements file requirements = temp_reqs remote = True @@ -1777,7 +1810,7 @@ def do_install( error, traceback = None, None click.echo( crayons.normal( - u"Requirements file provided! Importing into Pipfile…", bold=True + fix_utf8("Requirements file provided! Importing into Pipfile…"), bold=True ), err=True, ) @@ -1800,16 +1833,15 @@ def do_install( finally: # If requirements file was provided by remote url delete the temporary file if remote: - os.close(fd) # Close for windows to allow file cleanup. - os.remove(project.path_to(temp_reqs)) + fd.close() # Close for windows to allow file cleanup. + os.remove(temp_reqs) if error and traceback: click.echo(crayons.red(error)) click.echo(crayons.blue(str(traceback)), err=True) - requirements_directory.cleanup() sys.exit(1) if code: click.echo( - crayons.normal(u"Discovering imports from local codebase…", bold=True) + crayons.normal(fix_utf8("Discovering imports from local codebase…"), bold=True) ) for req in import_from_code(code): click.echo(" Found {0}!".format(crayons.green(req))) @@ -1886,17 +1918,21 @@ def do_install( for pkg_line in pkg_list: click.echo( crayons.normal( - u"Installing {0}…".format(crayons.green(pkg_line, bold=True)), + fix_utf8("Installing {0}…".format(crayons.green(pkg_line, bold=True))), bold=True, ) ) # pip install: - with spinner(): + with vistir.contextmanagers.temp_environ(), spinner(text="Installing...", + spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: + if "PIP_USER" in os.environ: + del os.environ["PIP_USER"] try: pkg_requirement = Requirement.from_line(pkg_line) except ValueError as e: - click.echo("{0}: {1}".format(crayons.red("WARNING"), e)) - requirements_directory.cleanup() + sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) sys.exit(1) if index_url: pkg_requirement.index = index_url @@ -1907,14 +1943,14 @@ def do_install( selective_upgrade=selective_upgrade, no_deps=False, pre=pre, - requirements_dir=requirements_directory.name, + requirements_dir=requirements_directory, index=index_url, extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) # Warn if --editable wasn't passed. if pkg_requirement.is_vcs and not pkg_requirement.editable: - click.echo( + sp.write_err( "{0}: You installed a VCS dependency in non-editable mode. " "This will work fine, but sub-dependencies will not be resolved by {1}." "\n To enable this sub-dependency functionality, specify that this dependency is editable." @@ -1923,37 +1959,34 @@ def do_install( crayons.red("$ pipenv lock"), ) ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. - try: - assert c.return_code == 0 - except AssertionError: - click.echo( - "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(pkg_line) - ), - err=True, - ) - click.echo(crayons.blue(format_pip_error(c.err)), err=True) - if "setup.py egg_info" in c.err: - click.echo( - "This is likely caused by a bug in {0}. " - "Report this to its maintainers.".format( - crayons.green(pkg_requirement.name) + click.echo(crayons.blue(format_pip_output(c.out))) + # Ensure that package was successfully installed. + if c.return_code != 0: + sp.write_err(vistir.compat.fs_str( + "{0} An error occurred while installing {1}!".format( + crayons.red("Error: ", bold=True), crayons.green(pkg_line) ), - err=True, + )) + sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) + if "setup.py egg_info" in c.err: + sp.write_err(vistir.compat.fs_str( + "This is likely caused by a bug in {0}. " + "Report this to its maintainers.".format( + crayons.green(pkg_requirement.name) + ) + )) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sys.exit(1) + sp.write(vistir.compat.fs_str( + u"{0} {1} {2} {3}{4}".format( + crayons.normal(u"Adding", bold=True), + crayons.green(u"{0}".format(pkg_requirement.name), bold=True), + crayons.normal(u"to Pipfile's", bold=True), + crayons.red(u"[dev-packages]" if dev else u"[packages]", bold=True), + crayons.normal(fix_utf8("…"), bold=True), ) - requirements_directory.cleanup() - sys.exit(1) - click.echo( - "{0} {1} {2} {3}{4}".format( - crayons.normal("Adding", bold=True), - crayons.green(pkg_requirement.name, bold=True), - crayons.normal("to Pipfile's", bold=True), - crayons.red("[dev-packages]" if dev else "[packages]", bold=True), - crayons.normal("…", bold=True), - ) - ) + )) + sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Installation Succeeded")) # Add the package to the Pipfile. try: project.add_package_to_pipfile(pkg_requirement, dev) @@ -1975,7 +2008,6 @@ def do_install( pypi_mirror=pypi_mirror, skip_lock=skip_lock, ) - requirements_directory.cleanup() sys.exit(0) @@ -2009,7 +2041,7 @@ def do_uninstall( # Un-install all dependencies, if --all was provided. if all is True: click.echo( - crayons.normal(u"Un-installing all packages from virtualenv…", bold=True) + crayons.normal(fix_utf8("Un-installing all packages from virtualenv…"), bold=True) ) do_purge(allow_global=system) return @@ -2025,7 +2057,7 @@ def do_uninstall( return click.echo( crayons.normal( - u"Un-installing {0}…".format(crayons.red("[dev-packages]")), bold=True + fix_utf8("Un-installing {0}…".format(crayons.red("[dev-packages]"))), bold=True ) ) package_names = project.dev_packages.keys() @@ -2033,7 +2065,7 @@ def do_uninstall( click.echo(crayons.red("No package provided!"), err=True) return 1 for package_name in package_names: - click.echo(u"Un-installing {0}…".format(crayons.green(package_name))) + click.echo(fix_utf8("Un-installing {0}…".format(crayons.green(package_name)))) cmd = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=system)), package_name ) @@ -2055,7 +2087,7 @@ def do_uninstall( continue click.echo( - u"Removing {0} from Pipfile…".format(crayons.green(package_name)) + fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name))) ) # Remove package from both packages and dev-packages. project.remove_package_from_pipfile(package_name, dev=True) @@ -2076,7 +2108,7 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror from .shells import choose_shell shell = choose_shell() - click.echo("Launching subshell in virtual environment…", err=True) + click.echo(fix_utf8("Launching subshell in virtual environment…"), err=True) fork_args = (project.virtualenv_location, project.project_directory, shell_args) @@ -2087,9 +2119,9 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror try: shell.fork_compat(*fork_args) except (AttributeError, ImportError): - click.echo( - u"Compatibility mode not supported. " - u"Trying to continue as well-configured shell…", + click.echo(fix_utf8( + "Compatibility mode not supported. " + "Trying to continue as well-configured shell…"), err=True, ) shell.fork(*fork_args) @@ -2099,11 +2131,11 @@ def _inline_activate_virtualenv(): try: activate_this = which("activate_this.py") if not activate_this or not os.path.exists(activate_this): - click.echo( - u"{0}: activate_this.py not found. Your environment is most " - u"certainly not activated. Continuing anyway…" - u"".format(crayons.red("Warning", bold=True)), - err=True, + click.echo(fix_utf8( + "{0}: activate_this.py not found. Your environment is most " + "certainly not activated. Continuing anyway…").format( + crayons.red("Warning", bold=True) + ), err=True, ) return with open(activate_this) as f: @@ -2267,7 +2299,7 @@ def do_check( sys.exit(1) else: sys.exit(0) - click.echo(crayons.normal(u"Checking PEP 508 requirements…", bold=True)) + click.echo(crayons.normal(fix_utf8("Checking PEP 508 requirements…"), bold=True)) if system: python = system_which("python") else: @@ -2303,7 +2335,7 @@ def do_check( sys.exit(1) else: click.echo(crayons.green("Passed!")) - click.echo(crayons.normal(u"Checking installed package safety…", bold=True)) + click.echo(crayons.normal(fix_utf8("Checking installed package safety…"), bold=True)) path = pep508checker.__file__.rstrip("cdo") path = os.sep.join(__file__.split(os.sep)[:-1] + ["patched", "safety.zip"]) if not system: @@ -2501,7 +2533,7 @@ def do_sync( ) # Install everything. - requirements_dir = vistir.compat.TemporaryDirectory( + requirements_dir = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) do_init( @@ -2513,7 +2545,6 @@ def do_sync( deploy=deploy, system=system, ) - requirements_dir.cleanup() click.echo(crayons.green("All dependencies are now up-to-date!")) @@ -2548,7 +2579,7 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro else: click.echo( crayons.white( - "Uninstalling {0}…".format(repr(apparent_bad_package)), bold=True + fix_utf8("Uninstalling {0}…".format(repr(apparent_bad_package))), bold=True ) ) # Uninstall the package. diff --git a/pipenv/environments.py b/pipenv/environments.py index e6de4f0d..c5874136 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -1,7 +1,9 @@ +# -*- coding=utf-8 -*- + import os import sys from appdirs import user_cache_dir -from .vendor.vistir.misc import fs_str +from .vendor.vistir.misc import fs_str, to_text # HACK: avoid resolver.py uses the wrong byte code files. @@ -259,3 +261,8 @@ def is_verbose(threshold=1): def is_quiet(threshold=-1): return PIPENV_VERBOSITY <= threshold + + +PIPENV_SPINNER_FAIL_TEXT = fs_str(to_text(u"✘ {0}")) if not PIPENV_HIDE_EMOJIS else ("{0}") + +PIPENV_SPINNER_OK_TEXT = fs_str(to_text(u"✔ {0}")) if not PIPENV_HIDE_EMOJIS else ("{0}") diff --git a/pipenv/project.py b/pipenv/project.py index c2dc9668..280a6f8b 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -10,6 +10,7 @@ import hashlib import contoml from first import first from cached_property import cached_property +import operator import pipfile import pipfile.api import six @@ -144,7 +145,7 @@ class Project(object): self._lockfile_newlines = DEFAULT_NEWLINES self._requirements_location = None self._original_dir = os.path.abspath(os.curdir) - self.which = which + self._which = which self.python_version = python_version # Hack to skip this during pipenv run, or -r. if ("run" not in sys.argv) and chdir: @@ -678,6 +679,54 @@ class Project(object): data[u"requires"] = {"python_version": version[: len("2.7")]} self.write_toml(data, "Pipfile") + def get_or_create_lockfile(self): + from requirementslib.models.lockfile import Lockfile as Req_Lockfile + lockfile = None + try: + lockfile = Req_Lockfile.load(self.lockfile_location) + except OSError: + lockfile = Req_Lockfile(self.lockfile_content) + return lockfile + else: + if lockfile._lockfile is not None: + return lockfile + if self.lockfile_exists and self.lockfile_content: + from .vendor.plette.lockfiles import Lockfile + lockfile_dict = self.lockfile_content.copy() + sources = lockfile_dict["_meta"].get("sources", []) + if not sources: + sources = self.pipfile_sources + elif not isinstance(sources, list): + sources = [sources,] + lockfile_dict["_meta"]["sources"] = [ + { + "name": s["name"], + "url": s["url"], + "verify_ssl": ( + s["verify_ssl"] if isinstance(s["verify_ssl"], bool) else ( + True if s["verify_ssl"].lower() == "true" else False + ) + ) + } for s in sources + ] + _created_lockfile = Lockfile(lockfile_dict) + lockfile._lockfile = lockfile.projectfile.model = _created_lockfile + return lockfile + elif self.pipfile_exists: + from .vendor.plette.lockfiles import Lockfile, PIPFILE_SPEC_CURRENT + lockfile_dict = { + "_meta": { + "hash": {"sha256": self.calculate_pipfile_hash()}, + "pipfile-spec": PIPFILE_SPEC_CURRENT, + "sources": self.pipfile_sources, + "requires": self.parsed_pipfile.get("requires", {}) + }, + "default": self._lockfile["default"].copy(), + "develop": self._lockfile["develop"].copy() + } + lockfile._lockfile = Lockfile(lockfile_dict) + return lockfile + def write_toml(self, data, path=None): """Writes the given data structure out as TOML.""" if path is None: @@ -939,6 +988,8 @@ class Project(object): location = self.virtualenv_location if self.virtualenv_location else sys.prefix prefix = vistir.compat.Path(location).as_posix() scheme = sysconfig._get_default_scheme() + if not scheme: + scheme = "posix_prefix" if not sys.platform == "win32" else "nt" config = { "base": prefix, "installed_base": prefix, @@ -953,3 +1004,26 @@ class Project(object): if "prefix" not in paths: paths["prefix"] = prefix return paths + + @cached_property + def finders(self): + from .vendor.pythonfinder import Finder + finders = [ + Finder(path=self.env_paths["scripts"], global_search=gs, system=False) + for gs in (False, True) + ] + return finders + + @property + def finder(self): + return next(iter(self.finders), None) + + def which(self, search, as_path=True): + find = operator.methodcaller("which", search) + result = next(iter(filter(None, (find(finder) for finder in self.finders))), None) + if not result: + result = self._which(search) + else: + if as_path: + result = str(result.path) + return result diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 8c282b71..2854a93a 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -7,53 +7,52 @@ os.environ["PIP_PYTHON_PATH"] = sys.executable def _patch_path(): + import site pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + pipenv_site_dir = os.path.dirname(pipenv_libdir) + site.addsitedir(pipenv_site_dir) for _dir in ("vendor", "patched"): sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) - site_packages_dir = os.path.dirname(pipenv_libdir) - if site_packages_dir not in sys.path: - sys.path.append(site_packages_dir) + + +def get_parser(): + from argparse import ArgumentParser + parser = ArgumentParser("pipenvresolver") + parser.add_argument("--pre", action="store_true", default=False) + parser.add_argument("--clear", action="store_true", default=False) + parser.add_argument("--verbose", "-v", action="count", default=False) + parser.add_argument("--debug", action="store_true", default=False) + parser.add_argument("--system", action="store_true", default=False) + parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store", + default=os.environ.get("PIPENV_REQ_DIR")) + parser.add_argument("packages", nargs="*") + return parser def which(*args, **kwargs): return sys.executable -def main(): - do_pre = "--pre" in " ".join(sys.argv) - do_clear = "--clear" in " ".join(sys.argv) - is_verbose = "--verbose" in " ".join(sys.argv) - is_debug = "--debug" in " ".join(sys.argv) - system = "--system" in " ".join(sys.argv) - new_sys_argv = [] - for v in sys.argv: - if v.startswith("--"): - continue +def handle_parsed_args(parsed): + if parsed.debug: + parsed.verbose = max(parsed.verbose, 2) + if parsed.verbose > 1: + logging.getLogger("notpip").setLevel(logging.DEBUG) + elif parsed.verbose > 0: + logging.getLogger("notpip").setLevel(logging.INFO) + if "PIPENV_PACKAGES" in os.environ: + parsed.packages += os.environ["PIPENV_PACKAGES"].strip().split("\n") + return parsed - else: - new_sys_argv.append(v) - sys.argv = new_sys_argv +def main(pre, clear, verbose, system, requirements_dir, packages): os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) os.environ["PIP_PYTHON_PATH"] = sys.executable - verbosity = int(os.environ.get("PIPENV_VERBOSITY", 0)) - if is_debug: - verbosity = max(verbosity, 2) - elif is_verbose: - verbosity = max(verbosity, 1) - if verbosity > 1: # Shit's getting real at this point. - logging.getLogger("notpip").setLevel(logging.DEBUG) - elif verbosity > 0: - logging.getLogger("notpip").setLevel(logging.INFO) + import warnings + from pipenv.vendor.vistir.compat import ResourceWarning + warnings.filterwarnings("ignore", category=ResourceWarning) - if "PIPENV_PACKAGES" in os.environ: - packages = os.environ["PIPENV_PACKAGES"].strip().split("\n") - else: - packages = sys.argv[1:] - for i, package in enumerate(packages): - if package.startswith("--"): - del packages[i] from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources pypi_mirror_source = ( @@ -62,7 +61,7 @@ def main(): else None ) - def resolve(packages, pre, project, sources, clear, system): + def resolve(packages, pre, project, sources, clear, system, requirements_dir=None): return resolve_deps( packages, which, @@ -71,6 +70,7 @@ def main(): sources=sources, clear=clear, allow_global=system, + req_dir=requirements_dir ) from pipenv.core import project @@ -82,19 +82,31 @@ def main(): ) results = resolve( packages, - pre=do_pre, + pre=pre, project=project, sources=sources, - clear=do_clear, + clear=clear, system=system, + requirements_dir=requirements_dir, ) print("RESULTS:") if results: - print(json.dumps(results)) + import traceback + if isinstance(results, (Exception, traceback.types.TracebackType)): + sys.stderr.write(traceback.print_tb(results)) + sys.stderr.write(sys.exc_value()) + else: + print(json.dumps(results)) else: print(json.dumps([])) if __name__ == "__main__": + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" _patch_path() - main() + parser = get_parser() + parsed, remaining = parser.parse_known_intermixed_args() + sys.argv = remaining + parsed = handle_parsed_args(parsed) + main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.requirements_dir, + parsed.packages) diff --git a/pipenv/utils.py b/pipenv/utils.py index b965e46d..d8f2432b 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -16,6 +16,11 @@ from click import echo as click_echo from first import first from vistir.misc import fs_str +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +from six.moves import Mapping + +from vistir.compat import ResourceWarning + try: from weakref import finalize except ImportError: @@ -38,14 +43,8 @@ from contextlib import contextmanager from . import environments from .pep508checker import lookup -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) from six.moves.urllib.parse import urlparse -from six.moves import Mapping - -if six.PY2: - - class ResourceWarning(Warning): - pass +from urllib3 import util as urllib3_util specifiers = [k for k in lookup.keys()] @@ -127,20 +126,14 @@ def parse_python_version(output): def python_version(path_to_python): - import delegator + from .vendor.pythonfinder.utils import get_python_version if not path_to_python: return None try: - c = delegator.run([path_to_python, "--version"], block=False) + version = get_python_version(path_to_python) except Exception: return None - c.block() - version = parse_python_version(c.out.strip() or c.err.strip()) - try: - version = u"{major}.{minor}.{micro}".format(**version) - except TypeError: - return None return version @@ -194,7 +187,7 @@ def prepare_pip_source_args(sources, pip_args=None): # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urlparse(sources[0]["url"]).hostname] + ["--trusted-host", urllib3_util.parse_url(sources[0]["url"]).host] ) # Add additional sources as extra indexes. if len(sources) > 1: @@ -203,7 +196,7 @@ def prepare_pip_source_args(sources, pip_args=None): # Trust the host if it's not verified. if not source.get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urlparse(source["url"]).hostname] + ["--trusted-host", urllib3_util.parse_url(source["url"]).host] ) return pip_args @@ -228,7 +221,7 @@ def actually_resolve_deps( from pipenv.patched.piptools import logging as piptools_logging from pipenv.patched.piptools.exceptions import NoCandidateFound from .vendor.requirementslib.models.requirements import Requirement - from ._compat import TemporaryDirectory, NamedTemporaryFile + from .vendor.vistir.path import create_tracked_tempdir, create_tracked_tempfile class PipCommand(basecommand.Command): """Needed for pip-tools.""" @@ -236,10 +229,8 @@ def actually_resolve_deps( name = "PipCommand" constraints = [] - cleanup_req_dir = False if not req_dir: - req_dir = TemporaryDirectory(suffix="-requirements", prefix="pipenv-") - cleanup_req_dir = True + req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") for dep in deps: if not dep: continue @@ -267,26 +258,26 @@ def actually_resolve_deps( if sources: pip_args = prepare_pip_source_args(sources, pip_args) if environments.is_verbose(): - print("Using pip: {0}".format(" ".join(pip_args))) - with NamedTemporaryFile( + click_echo(crayons.blue("Using pip: {0}".format(" ".join(pip_args)))) + constraints_file = create_tracked_tempfile( mode="w", prefix="pipenv-", suffix="-constraints.txt", - dir=req_dir.name, + dir=req_dir, delete=False, - ) as f: - if sources: - requirementstxt_sources = " ".join(pip_args) if pip_args else "" - requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") - f.write(u"{0}\n".format(requirementstxt_sources)) - f.write(u"\n".join([_constraint for _constraint in constraints])) - constraints_file = f.name + ) + if sources: + requirementstxt_sources = " ".join(pip_args) if pip_args else "" + requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") + constraints_file.write(u"{0}\n".format(requirementstxt_sources)) + constraints_file.write(u"\n".join([_constraint for _constraint in constraints])) + constraints_file.close() pip_options, _ = pip_command.parser.parse_args(pip_args) pip_options.cache_dir = environments.PIPENV_CACHE_DIR session = pip_command._build_session(pip_options) pypi = PyPIRepository(pip_options=pip_options, use_json=False, session=session) constraints = parse_requirements( - constraints_file, finder=pypi.finder, session=pypi.session, options=pip_options + constraints_file.name, finder=pypi.finder, session=pypi.session, options=pip_options ) constraints = [c for c in constraints] if environments.is_verbose(): @@ -326,11 +317,7 @@ def actually_resolve_deps( "Please check your version specifier and version number. See PEP440 for more information." ) ) - if cleanup_req_dir: - req_dir.cleanup() raise RuntimeError - if cleanup_req_dir: - req_dir.cleanup() return (resolved_tree, hashes, markers_lookup, resolver) @@ -343,43 +330,72 @@ def venv_resolve_deps( allow_global=False, pypi_mirror=None, ): - from .vendor.vistir.misc import fs_str + from .vendor.vistir.misc import fs_str, run + from .vendor.vistir.compat import Path + from .vendor.vistir.path import create_tracked_tempdir + from .cmdparse import Script + from .core import spinner + from .vendor.pexpect.exceptions import EOF from .vendor import delegator from . import resolver import json if not deps: return [] - resolver = escape_grouped_arguments(resolver.__file__.rstrip("co")) - cmd = "{0} {1} {2} {3} {4}".format( - escape_grouped_arguments(which("python", allow_global=allow_global)), - resolver, - "--pre" if pre else "", - "--clear" if clear else "", - "--system" if allow_global else "", - ) + + req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") + + cmd = [ + which("python", allow_global=allow_global), + Path(resolver.__file__.rstrip("co")).as_posix() + ] + if pre: + cmd.append("--pre") + if clear: + cmd.append("--clear") + if allow_global: + cmd.append("--system") with temp_environ(): os.environ = {fs_str(k): fs_str(val) for k, val in os.environ.items()} os.environ["PIPENV_PACKAGES"] = str("\n".join(deps)) if pypi_mirror: os.environ["PIPENV_PYPI_MIRROR"] = str(pypi_mirror) os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) - c = delegator.run(cmd, block=True) - try: - assert c.return_code == 0 - except AssertionError: - if environments.is_verbose(): - click_echo(c.out, err=True) - click_echo(c.err, err=True) - else: - click_echo(c.err[(int(len(c.err) / 2) - 1):], err=True) - sys.exit(c.return_code) + os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) + os.environ["PIP_NO_INPUT"] = fs_str("1") + + out = "" + EOF.__module__ = "pexpect.exceptions" + with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: + c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) + _out = u"" + while True: + result = c.expect(u"\n", timeout=-1) + if result is EOF or result is None: + break + _out = c.out + out += _out + sp.text = fs_str("Locking... {0}".format(_out[:100])) + if environments.is_verbose(): + sp.write_err(_out.rstrip()) + c.block() + if c.return_code != 0: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Locking Failed!" + )) + click_echo(c.err.strip(), err=True) + sys.exit(c.return_code) + else: + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) if environments.is_verbose(): click_echo(c.out.split("RESULTS:")[0], err=True) try: return json.loads(c.out.split("RESULTS:")[1].strip()) except IndexError: + click_echo(c.out.strip()) + click_echo(c.err.strip(), err=True) raise RuntimeError("There was a problem with locking.") @@ -392,13 +408,13 @@ def resolve_deps( clear=False, pre=False, allow_global=False, + req_dir=None ): """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip. """ from .patched.notpip._vendor.requests.exceptions import ConnectionError from .vendor.requirementslib.models.requirements import Requirement - from ._compat import TemporaryDirectory index_lookup = {} markers_lookup = {} @@ -408,7 +424,10 @@ def resolve_deps( if not deps: return results # First (proper) attempt: - req_dir = TemporaryDirectory(prefix="pipenv-", suffix="-requirements") + req_dir = req_dir if req_dir else os.environ.get("req_dir", None) + if not req_dir: + from .vendor.vistir.path import create_tracked_tempdir + req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-requirements") with HackedPythonVersion(python_version=python, python_path=python_path): try: resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( @@ -444,7 +463,6 @@ def resolve_deps( req_dir=req_dir, ) except RuntimeError: - req_dir.cleanup() sys.exit(1) for result in resolved_tree: if not result.editable: @@ -503,7 +521,6 @@ def resolve_deps( entry.update({"markers": markers_lookup.get(result.name)}) entry = translate_markers(entry) results.append(entry) - req_dir.cleanup() return results @@ -526,7 +543,6 @@ def is_pinned(val): def convert_deps_to_pip(deps, project=None, r=True, include_index=True): """"Converts a Pipfile-formatted dependency to a pip-formatted one.""" - from ._compat import NamedTemporaryFile from .vendor.requirementslib.models.requirements import Requirement dependencies = [] @@ -541,7 +557,8 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=True): return dependencies # Write requirements.txt to tmp directory. - f = NamedTemporaryFile(suffix="-requirements.txt", delete=False) + from .vendor.vistir.path import create_tracked_tempfile + f = create_tracked_tempfile(suffix="-requirements.txt", delete=False) f.write("\n".join(dependencies).encode("utf-8")) f.close() return f.name @@ -1054,54 +1071,6 @@ def escape_cmd(cmd): return cmd -@contextmanager -def atomic_open_for_write(target, binary=False, newline=None, encoding=None): - """Atomically open `target` for writing. - - This is based on Lektor's `atomic_open()` utility, but simplified a lot - to handle only writing, and skip many multi-process/thread edge cases - handled by Werkzeug. - - How this works: - - * Create a temp file (in the same directory of the actual target), and - yield for surrounding code to write to it. - * If some thing goes wrong, try to remove the temp file. The actual target - is not touched whatsoever. - * If everything goes well, close the temp file, and replace the actual - target with this new file. - """ - from ._compat import NamedTemporaryFile - - mode = "w+b" if binary else "w" - f = NamedTemporaryFile( - dir=os.path.dirname(target), - prefix=".__atomic-write", - mode=mode, - encoding=encoding, - newline=newline, - delete=False, - ) - # set permissions to 0644 - os.chmod(f.name, stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - try: - yield f - except BaseException: - f.close() - try: - os.remove(f.name) - except OSError: - pass - raise - else: - f.close() - try: - os.remove(target) # This is needed on Windows. - except OSError: - pass - os.rename(f.name, target) # No os.replace() on Python 2. - - def safe_expandvars(value): """Call os.path.expandvars if value is a string, otherwise do nothing. """ @@ -1127,8 +1096,8 @@ def get_vcs_deps( dev=False, pypi_mirror=None, ): - from ._compat import TemporaryDirectory, Path - import atexit + from .vendor.vistir.compat import Path + from .vendor.vistir.path import create_tracked_tempdir from .vendor.requirementslib.models.requirements import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" @@ -1144,8 +1113,7 @@ def get_vcs_deps( ) src_dir.mkdir(mode=0o775, exist_ok=True) else: - src_dir = TemporaryDirectory(prefix="pipenv-lock-dir") - atexit.register(src_dir.cleanup) + src_dir = create_tracked_tempdir(prefix="pipenv-lock-dir") for pkg_name, pkg_pipfile in packages.items(): requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) name = requirement.normalized_name @@ -1280,7 +1248,6 @@ def is_virtual_environment(path): @contextmanager def locked_repository(requirement): from .vendor.vistir.path import create_tracked_tempdir - from .vendor.vistir.misc import fs_str src_dir = create_tracked_tempdir(prefix="pipenv-src") if not requirement.is_vcs: return diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py index 3ffb2e31..56d12458 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py @@ -7,6 +7,8 @@ import locale import errno from pexpect.popen_spawn import PopenSpawn +import pexpect +pexpect.EOF.__module__ = "pexpect.exceptions" # Include `unicode` in STR_TYPES for Python 2.X try: @@ -110,7 +112,7 @@ class Command(object): if self.subprocess.before: result += self.subprocess.before - if self.subprocess.after: + if self.subprocess.after and self.subprocess.after is not pexpect.EOF: result += self.subprocess.after result += self.subprocess.read() @@ -206,7 +208,10 @@ class Command(object): if self.blocking: raise RuntimeError("expect can only be used on non-blocking commands.") - self.subprocess.expect(pattern=pattern, timeout=timeout) + try: + self.subprocess.expect(pattern=pattern, timeout=timeout) + except pexpect.EOF: + pass def send(self, s, end=os.linesep, signal=False): """Sends the given string or signal to std_in.""" @@ -249,8 +254,11 @@ class Command(object): self.subprocess.wait() else: self.subprocess.sendeof() - self.subprocess.wait() - self.subprocess.proc.stdout.close() + try: + self.subprocess.wait() + finally: + if self.subprocess.proc.stdout: + self.subprocess.proc.stdout.close() def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next @@ -272,7 +280,6 @@ class Command(object): c.run(block=False, cwd=cwd) if data: c.send(data) - c.subprocess.sendeof() c.block() return c diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 9ac6031c..91e9cabb 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.3' +__version__ = '1.1.3.post1' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index d54393b6..f39299a3 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -341,7 +341,6 @@ class SystemPath(object): self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) else: self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver] - print(ver) return ver @classmethod diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index e0bc6746..7b4b6376 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,6 +1,9 @@ # -*- coding=utf-8 -*- -__version__ = '1.1.10' +__version__ = '1.2.0' + +from .models.requirements import Requirement +from .models.lockfile import Lockfile +from .models.pipfile import Pipfile -from .exceptions import RequirementError -from .models import Requirement, Lockfile, Pipfile +__all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/vendor/requirementslib/exceptions.py index 82578624..de8bf8ef 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/vendor/requirementslib/exceptions.py @@ -9,6 +9,8 @@ if six.PY2: def __init__(self, *args, **kwargs): self.errno = errno.EEXIST super(FileExistsError, self).__init__(*args, **kwargs) +else: + from six.moves.builtins import FileExistsError class RequirementError(Exception): diff --git a/pipenv/vendor/requirementslib/models/cache.py b/pipenv/vendor/requirementslib/models/cache.py index 16fc4ba8..71701090 100644 --- a/pipenv/vendor/requirementslib/models/cache.py +++ b/pipenv/vendor/requirementslib/models/cache.py @@ -5,21 +5,19 @@ import copy import hashlib import json import os -import six import sys import requests -import pip_shims import vistir from appdirs import user_cache_dir +from pip_shims.shims import FAVORITE_HASH, SafeFileCache from packaging.requirements import Requirement from .utils import as_tuple, key_from_req, lookup_table, get_pinned_version - -if six.PY2: - from ..exceptions import FileExistsError +from ..exceptions import FileExistsError +from ..utils import VCS_SUPPORT CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) @@ -189,7 +187,7 @@ class DependencyCache(object): for dep_name in self.cache[name][version_and_extras]) -class HashCache(pip_shims.SafeFileCache): +class HashCache(SafeFileCache): """Caches hashes of PyPI artifacts so we do not need to re-download them. Hashes are only cached when the URL appears to contain a hash in it and the @@ -206,7 +204,7 @@ class HashCache(pip_shims.SafeFileCache): def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs = pip_shims.VcsSupport() + vcs = VCS_SUPPORT orig_scheme = location.scheme new_location = copy.deepcopy(location) if orig_scheme in vcs.all_schemes: @@ -223,11 +221,11 @@ class HashCache(pip_shims.SafeFileCache): return hash_value.decode('utf8') def _get_file_hash(self, location): - h = hashlib.new(pip_shims.FAVORITE_HASH) + h = hashlib.new(FAVORITE_HASH) with vistir.contextmanagers.open_file(location, self.session) as fp: for chunk in iter(lambda: fp.read(8096), b""): h.update(chunk) - return ":".join([pip_shims.FAVORITE_HASH, h.hexdigest()]) + return ":".join([FAVORITE_HASH, h.hexdigest()]) class _JSONCache(object): diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index 1997fc1f..3e482813 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -5,6 +5,7 @@ import copy import os import attr +import itertools import plette.lockfiles import six @@ -50,6 +51,31 @@ class Lockfile(object): def _get_lockfile(self): return self.projectfile.lockfile + @property + def lockfile(self): + return self._lockfile + + @property + def section_keys(self): + return ["default", "develop"] + + @property + def extended_keys(self): + return [k for k in itertools.product(self.section_keys, ["", "vcs", "editable"])] + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_lockfile = k in self.extended_keys or self.lockfile.__contains__(k) + if check_lockfile: + return True + return super(Lockfile, self).__contains__(k) + + def __setitem__(self, k, v): + lockfile = self._lockfile + lockfile.__setitem__(k, v) + def __getitem__(self, k, *args, **kwargs): retval = None lockfile = self._lockfile diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 0d1c04c8..fe7743c2 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -75,11 +75,19 @@ class Pipfile(object): def get_deps(self, dev=False, only=True): deps = {} if dev: - deps.update(self.pipfile["dev-packages"]._data) + deps.update(self.pipfile._data["dev-packages"]) if only: return deps - deps = merge_items([deps, self.pipfile["packages"]._data]) - return deps + return merge_items([deps, self.pipfile._data["packages"]]) + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) + if check_pipfile: + return True + return super(Pipfile, self).__contains__(k) def __getitem__(self, k, *args, **kwargs): retval = None diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index db004869..3a029cbc 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -16,10 +16,7 @@ from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement from packaging.specifiers import Specifier, SpecifierSet from packaging.utils import canonicalize_name -from pip_shims.shims import ( - InstallRequirement, Link, Wheel, _strip_extras, parse_version, path_to_url, - url_to_path -) +from pip_shims.shims import _strip_extras, parse_version, path_to_url, url_to_path from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote from vistir.compat import FileNotFoundError, Path @@ -32,21 +29,16 @@ from vistir.path import ( from ..exceptions import RequirementError from ..utils import VCS_LIST, is_installable_file, is_vcs, ensure_setup_py from .baserequirement import BaseRequirement -from .dependencies import ( - AbstractDependency, find_all_matches, get_abstract_dependencies, - get_dependencies, get_finder -) from .markers import PipenvMarkers from .utils import ( HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string, filter_none, format_requirement, get_version, init_requirement, - is_pinned_requirement, make_install_requirement, optional_instance_of, - parse_extras, specs_to_string, split_markers_from_line, + is_pinned_requirement, make_install_requirement, optional_instance_of, parse_extras, + specs_to_string, split_markers_from_line, ireq_from_editable, ireq_from_line, split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs, normalize_name, + validate_specifiers, validate_vcs, normalize_name, create_link, Requirement as PkgResourcesRequirement ) -from .vcs import VCSRepository @attr.s @@ -128,7 +120,7 @@ class FileRequirement(BaseRequirement): editable = attr.ib(default=False, type=bool) extras = attr.ib(default=attr.Factory(list), type=list) uri = attr.ib(type=six.string_types) - link = attr.ib(type=Link) + link = attr.ib() name = attr.ib(type=six.string_types) req = attr.ib(type=PkgResourcesRequirement) _has_hashed_name = False @@ -166,6 +158,7 @@ class FileRequirement(BaseRequirement): See `https://bugs.python.org/issue23505#msg277350`. """ + # Git allows `git@github.com...` lines that are not really URIs. # Add "ssh://" so we can parse correctly, and restore afterwards. fixed_line = add_ssh_scheme_to_git_uri(line) @@ -176,7 +169,7 @@ class FileRequirement(BaseRequirement): p = Path(fixed_line).absolute() path = p.as_posix() uri = p.as_uri() - link = Link(uri) + link = create_link(uri) try: relpath = get_converted_relative_path(path) except ValueError: @@ -225,7 +218,7 @@ class FileRequirement(BaseRequirement): uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. - link = Link( + link = create_link( urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme)) ) @@ -246,6 +239,7 @@ class FileRequirement(BaseRequirement): if self.link and self.link.egg_fragment: return self.link.egg_fragment elif self.link and self.link.is_wheel: + from pip_shims import Wheel return Wheel(self.link.filename).name if ( self._uri_scheme != "uri" @@ -263,7 +257,7 @@ class FileRequirement(BaseRequirement): except (FileNotFoundError, IOError) as e: dist = None except Exception as e: - from pip_shims.shims import InstallRequirement, make_abstract_dist + from pip_shims.shims import make_abstract_dist try: if not isinstance(Path, self.path): @@ -271,9 +265,9 @@ class FileRequirement(BaseRequirement): else: _path = self.path if self.editable: - _ireq = InstallRequirement.from_editable(_path.as_uri()) + _ireq = ireq_from_editable(_path.as_uri()) else: - _ireq = InstallRequirement.from_line(_path.as_posix()) + _ireq = ireq_from_line(_path.as_posix()) dist = make_abstract_dist(_ireq).get_dist() name = dist.project_name except (TypeError, ValueError, AttributeError) as e: @@ -286,7 +280,7 @@ class FileRequirement(BaseRequirement): self._has_hashed_name = True name = hashed_name if self.link and not self._has_hashed_name: - self.link = Link("{0}#egg={1}".format(self.link.url, name)) + self.link = create_link("{0}#egg={1}".format(self.link.url, name)) return name @link.default @@ -294,7 +288,7 @@ class FileRequirement(BaseRequirement): target = "{0}".format(self.uri) if hasattr(self, "name"): target = "{0}#egg={1}".format(target, self.name) - link = Link(target) + link = create_link(target) return link @req.default @@ -359,6 +353,7 @@ class FileRequirement(BaseRequirement): "uri_scheme": prefer, } if link and link.is_wheel: + from pip_shims import Wheel arg_dict["name"] = Wheel(link.filename).name elif link.egg_fragment: arg_dict["name"] = link.egg_fragment @@ -398,7 +393,7 @@ class FileRequirement(BaseRequirement): if not uri: uri = path_to_url(path) - link = Link(uri) + link = create_link(uri) arg_dict = { "name": name, @@ -588,6 +583,7 @@ class VCSRequirement(FileRequirement): return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) def get_vcs_repo(self, src_dir=None): + from .vcs import VCSRepository checkout_dir = self.get_checkout_dir(src_dir=src_dir) url = "{0}#egg={1}".format(self.vcs_uri, self.name) vcsrepo = VCSRepository( @@ -825,6 +821,7 @@ class Requirement(object): @classmethod def from_line(cls, line): + from pip_shims import InstallRequirement if isinstance(line, InstallRequirement): line = format_requirement(line) hashes = None @@ -1074,9 +1071,9 @@ class Requirement(object): if ireq_line.startswith("-e "): ireq_line = ireq_line[len("-e "):] with ensure_setup_py(self.req.path): - ireq = InstallRequirement.from_editable(ireq_line) + ireq = ireq_from_editable(ireq_line) else: - ireq = InstallRequirement.from_line(ireq_line) + ireq = ireq_from_line(ireq_line) if not getattr(ireq, "req", None): ireq.req = self.req.req else: @@ -1103,6 +1100,8 @@ class Requirement(object): :return: A set of requirement strings of the dependencies of this requirement. :rtype: set(str) """ + + from .dependencies import get_dependencies if not sources: sources = [{ 'name': 'pypi', @@ -1122,6 +1121,7 @@ class Requirement(object): :rtype: list[ :class:`~requirementslib.models.dependency.AbstractDependency` ] """ + from .dependencies import AbstractDependency, get_dependencies, get_abstract_dependencies if not self.abstract_dep: parent = getattr(self, 'parent', None) self.abstract_dep = AbstractDependency.from_requirement(self, parent=parent) @@ -1144,6 +1144,8 @@ class Requirement(object): :return: A list of Installation Candidates :rtype: list[ :class:`~pip._internal.index.InstallationCandidate` ] """ + + from .dependencies import get_finder, find_all_matches if not finder: finder = get_finder(sources=sources) return find_all_matches(finder, self.as_ireq()) diff --git a/pipenv/vendor/requirementslib/models/resolvers.py b/pipenv/vendor/requirementslib/models/resolvers.py index da6d0dda..1a239390 100644 --- a/pipenv/vendor/requirementslib/models/resolvers.py +++ b/pipenv/vendor/requirementslib/models/resolvers.py @@ -4,11 +4,10 @@ from contextlib import contextmanager import attr import six -from pip_shims.shims import VcsSupport, Wheel +from pip_shims.shims import Wheel -from ..utils import log +from ..utils import log, VCS_SUPPORT from .cache import HashCache -from .dependencies import AbstractDependency, find_all_matches, get_finder from .utils import format_requirement, is_pinned_requirement, version_from_ireq @@ -41,6 +40,7 @@ class DependencyResolver(object): @classmethod def create(cls, finder=None, allow_prereleases=False, get_all_hashes=True): if not finder: + from .dependencies import get_finder finder_args = [] if allow_prereleases: finder_args.append('--pre') @@ -140,6 +140,7 @@ class DependencyResolver(object): # Coerce input into AbstractDependency instances. # We accept str, Requirement, and AbstractDependency as input. + from .dependencies import AbstractDependency for dep in root_nodes: if isinstance(dep, six.string_types): dep = AbstractDependency.from_string(dep) @@ -183,6 +184,7 @@ class DependencyResolver(object): def get_hashes_for_one(self, ireq): if not self.finder: + from .dependencies import get_finder finder_args = [] if self.allow_prereleases: finder_args.append('--pre') @@ -191,7 +193,7 @@ class DependencyResolver(object): if ireq.editable: return set() - vcs = VcsSupport() + vcs = VCS_SUPPORT if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: return set() @@ -201,6 +203,7 @@ class DependencyResolver(object): matching_candidates = set() with self.allow_all_wheels(): + from .dependencies import find_all_matches matching_candidates = ( find_all_matches(self.finder, ireq, pre=self.allow_prereleases) ) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 7350d0ac..3103580e 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -19,7 +19,7 @@ from packaging.requirements import Requirement as PackagingRequirement from pkg_resources import Requirement from vistir.misc import dedup -from pip_shims.shims import InstallRequirement, Link + from ..utils import SCHEME_LIST, VCS_LIST, is_star @@ -37,6 +37,21 @@ def optional_instance_of(cls): return validators.optional(validators.instance_of(cls)) +def create_link(link): + from pip_shims import Link + return Link(link) + + +def ireq_from_line(ireq): + from pip_shims import InstallRequirement + return InstallRequirement.from_line(ireq) + + +def ireq_from_editable(ireq): + from pip_shims import InstallRequirement + return InstallRequirement.from_editable(ireq) + + def init_requirement(name): req = Requirement.parse(name) req.vcs = None @@ -92,7 +107,7 @@ def build_vcs_link(vcs, uri, name=None, ref=None, subdirectory=None, extras=None uri = "{0}{1}".format(uri, extras) if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) - return Link(uri) + return create_link(uri) def get_version(pipfile_entry): @@ -443,11 +458,11 @@ def make_install_requirement(name, version, extras, markers, constraint=False): extras_string = "[{}]".format(",".join(sorted(extras))) if not markers: - return InstallRequirement.from_line( + return ireq_from_line( str('{}{}=={}'.format(name, extras_string, version)), constraint=constraint) else: - return InstallRequirement.from_line( + return ireq_from_line( str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), constraint=constraint) diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index abc89831..085f3241 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -5,7 +5,6 @@ import contextlib import logging import os -import boltons.iterutils import six import tomlkit @@ -30,8 +29,10 @@ VCS_LIST = ("git", "svn", "hg", "bzr") VCS_SCHEMES = [] SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") +VCS_SUPPORT = VcsSupport() + if not VCS_SCHEMES: - VCS_SCHEMES = VcsSupport().all_schemes + VCS_SCHEMES = VCS_SUPPORT.all_schemes def setup_logger(): @@ -197,6 +198,127 @@ def ensure_setup_py(base_dir): setup_py.unlink() + +_UNSET = object() +_REMAP_EXIT = object() + + +# The following functionality is either borrowed or modified from the itertools module +# in the boltons library by Mahmoud Hashemi and distributed under the BSD license +# the text of which is included below: + +# (original text from https://github.com/mahmoud/boltons/blob/master/LICENSE) +# Copyright (c) 2013, Mahmoud Hashemi +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * The names of the contributors may not be used to endorse or +# promote products derived from this software without specific +# prior written permission. +# +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class PathAccessError(KeyError, IndexError, TypeError): + """An amalgamation of KeyError, IndexError, and TypeError, + representing what can occur when looking up a path in a nested + object. + """ + def __init__(self, exc, seg, path): + self.exc = exc + self.seg = seg + self.path = path + + def __repr__(self): + cn = self.__class__.__name__ + return '%s(%r, %r, %r)' % (cn, self.exc, self.seg, self.path) + + def __str__(self): + return ('could not access %r from path %r, got error: %r' + % (self.seg, self.path, self.exc)) + + +def get_path(root, path, default=_UNSET): + """Retrieve a value from a nested object via a tuple representing the + lookup path. + >>> root = {'a': {'b': {'c': [[1], [2], [3]]}}} + >>> get_path(root, ('a', 'b', 'c', 2, 0)) + 3 + The path format is intentionally consistent with that of + :func:`remap`. + One of get_path's chief aims is improved error messaging. EAFP is + great, but the error messages are not. + For instance, ``root['a']['b']['c'][2][1]`` gives back + ``IndexError: list index out of range`` + What went out of range where? get_path currently raises + ``PathAccessError: could not access 2 from path ('a', 'b', 'c', 2, + 1), got error: IndexError('list index out of range',)``, a + subclass of IndexError and KeyError. + You can also pass a default that covers the entire operation, + should the lookup fail at any level. + Args: + root: The target nesting of dictionaries, lists, or other + objects supporting ``__getitem__``. + path (tuple): A list of strings and integers to be successively + looked up within *root*. + default: The value to be returned should any + ``PathAccessError`` exceptions be raised. + """ + if isinstance(path, six.string_types): + path = path.split('.') + cur = root + try: + for seg in path: + try: + cur = cur[seg] + except (KeyError, IndexError) as exc: + raise PathAccessError(exc, seg, path) + except TypeError as exc: + # either string index in a list, or a parent that + # doesn't support indexing + try: + seg = int(seg) + cur = cur[seg] + except (ValueError, KeyError, IndexError, TypeError): + if not getattr(cur, "__iter__", None): + exc = TypeError('%r object is not indexable' + % type(cur).__name__) + raise PathAccessError(exc, seg, path) + except PathAccessError: + if default is _UNSET: + raise + return default + return cur + + +def default_visit(path, key, value): + return key, value + + +_orig_default_visit = default_visit + + # Modified from https://github.com/mahmoud/boltons/blob/master/boltons/iterutils.py def dict_path_enter(path, key, value): if isinstance(value, six.string_types): @@ -249,6 +371,157 @@ def dict_path_exit(path, key, old_parent, new_parent, new_items): return ret +def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, + **kwargs): + """The remap ("recursive map") function is used to traverse and + transform nested structures. Lists, tuples, sets, and dictionaries + are just a few of the data structures nested into heterogenous + tree-like structures that are so common in programming. + Unfortunately, Python's built-in ways to manipulate collections + are almost all flat. List comprehensions may be fast and succinct, + but they do not recurse, making it tedious to apply quick changes + or complex transforms to real-world data. + remap goes where list comprehensions cannot. + Here's an example of removing all Nones from some data: + >>> from pprint import pprint + >>> reviews = {'Star Trek': {'TNG': 10, 'DS9': 8.5, 'ENT': None}, + ... 'Babylon 5': 6, 'Dr. Who': None} + >>> pprint(remap(reviews, lambda p, k, v: v is not None)) + {'Babylon 5': 6, 'Star Trek': {'DS9': 8.5, 'TNG': 10}} + Notice how both Nones have been removed despite the nesting in the + dictionary. Not bad for a one-liner, and that's just the beginning. + See `this remap cookbook`_ for more delicious recipes. + .. _this remap cookbook: http://sedimental.org/remap.html + remap takes four main arguments: the object to traverse and three + optional callables which determine how the remapped object will be + created. + Args: + root: The target object to traverse. By default, remap + supports iterables like :class:`list`, :class:`tuple`, + :class:`dict`, and :class:`set`, but any object traversable by + *enter* will work. + visit (callable): This function is called on every item in + *root*. It must accept three positional arguments, *path*, + *key*, and *value*. *path* is simply a tuple of parents' + keys. *visit* should return the new key-value pair. It may + also return ``True`` as shorthand to keep the old item + unmodified, or ``False`` to drop the item from the new + structure. *visit* is called after *enter*, on the new parent. + The *visit* function is called for every item in root, + including duplicate items. For traversable values, it is + called on the new parent object, after all its children + have been visited. The default visit behavior simply + returns the key-value pair unmodified. + enter (callable): This function controls which items in *root* + are traversed. It accepts the same arguments as *visit*: the + path, the key, and the value of the current item. It returns a + pair of the blank new parent, and an iterator over the items + which should be visited. If ``False`` is returned instead of + an iterator, the value will not be traversed. + The *enter* function is only called once per unique value. The + default enter behavior support mappings, sequences, and + sets. Strings and all other iterables will not be traversed. + exit (callable): This function determines how to handle items + once they have been visited. It gets the same three + arguments as the other functions -- *path*, *key*, *value* + -- plus two more: the blank new parent object returned + from *enter*, and a list of the new items, as remapped by + *visit*. + Like *enter*, the *exit* function is only called once per + unique value. The default exit behavior is to simply add + all new items to the new parent, e.g., using + :meth:`list.extend` and :meth:`dict.update` to add to the + new parent. Immutable objects, such as a :class:`tuple` or + :class:`namedtuple`, must be recreated from scratch, but + use the same type as the new parent passed back from the + *enter* function. + reraise_visit (bool): A pragmatic convenience for the *visit* + callable. When set to ``False``, remap ignores any errors + raised by the *visit* callback. Items causing exceptions + are kept. See examples for more details. + remap is designed to cover the majority of cases with just the + *visit* callable. While passing in multiple callables is very + empowering, remap is designed so very few cases should require + passing more than one function. + When passing *enter* and *exit*, it's common and easiest to build + on the default behavior. Simply add ``from boltons.iterutils import + default_enter`` (or ``default_exit``), and have your enter/exit + function call the default behavior before or after your custom + logic. See `this example`_. + Duplicate and self-referential objects (aka reference loops) are + automatically handled internally, `as shown here`_. + .. _this example: http://sedimental.org/remap.html#sort_all_lists + .. _as shown here: http://sedimental.org/remap.html#corner_cases + """ + # TODO: improve argument formatting in sphinx doc + # TODO: enter() return (False, items) to continue traverse but cancel copy? + if not callable(visit): + raise TypeError('visit expected callable, not: %r' % visit) + if not callable(enter): + raise TypeError('enter expected callable, not: %r' % enter) + if not callable(exit): + raise TypeError('exit expected callable, not: %r' % exit) + reraise_visit = kwargs.pop('reraise_visit', True) + if kwargs: + raise TypeError('unexpected keyword arguments: %r' % kwargs.keys()) + + path, registry, stack = (), {}, [(None, root)] + new_items_stack = [] + while stack: + key, value = stack.pop() + id_value = id(value) + if key is _REMAP_EXIT: + key, new_parent, old_parent = value + id_value = id(old_parent) + path, new_items = new_items_stack.pop() + value = exit(path, key, old_parent, new_parent, new_items) + registry[id_value] = value + if not new_items_stack: + continue + elif id_value in registry: + value = registry[id_value] + else: + res = enter(path, key, value) + try: + new_parent, new_items = res + except TypeError: + # TODO: handle False? + raise TypeError('enter should return a tuple of (new_parent,' + ' items_iterator), not: %r' % res) + if new_items is not False: + # traverse unless False is explicitly passed + registry[id_value] = new_parent + new_items_stack.append((path, [])) + if value is not root: + path += (key,) + stack.append((_REMAP_EXIT, (key, new_parent, value))) + if new_items: + stack.extend(reversed(list(new_items))) + continue + if visit is _orig_default_visit: + # avoid function call overhead by inlining identity operation + visited_item = (key, value) + else: + try: + visited_item = visit(path, key, value) + except Exception: + if reraise_visit: + raise + visited_item = True + if visited_item is False: + continue # drop + elif visited_item is True: + visited_item = (key, value) + # TODO: typecheck? + # raise TypeError('expected (key, value) from visit(),' + # ' not: %r' % visited_item) + try: + new_items_stack[-1][1].append(visited_item) + except IndexError: + raise TypeError('expected remappable root, not: %r' % root) + return value + + def merge_items(target_list, sourced=False): if not sourced: target_list = [(id(t), t) for t in target_list] @@ -262,7 +535,7 @@ def merge_items(target_list, sourced=False): new_parent = ret try: - cur_val = boltons.iterutils.get_path(ret, path + (key,)) + cur_val = get_path(ret, path + (key,)) except KeyError as ke: pass else: @@ -279,9 +552,9 @@ def merge_items(target_list, sourced=False): source_map[path + (key,)] = t_name return True else: - remerge_visit = boltons.iterutils.default_visit + remerge_visit = default_visit - ret = boltons.iterutils.remap(target, enter=remerge_enter, visit=remerge_visit, + ret = remap(target, enter=remerge_enter, visit=remerge_visit, exit=remerge_exit) if not sourced: diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index abece020..781e2e98 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -18,7 +18,7 @@ from ._compat import unicode from ._utils import escape_string if PY2: - from functools32 import lru_cache + from pipenv.vendor.backports.functools_lru_cache import lru_cache else: from functools import lru_cache diff --git a/pipenv/vendor/tomlkit/toml_char.py b/pipenv/vendor/tomlkit/toml_char.py index 92f1a9c9..5164ea8b 100644 --- a/pipenv/vendor/tomlkit/toml_char.py +++ b/pipenv/vendor/tomlkit/toml_char.py @@ -4,7 +4,7 @@ from ._compat import PY2 from ._compat import unicode if PY2: - from functools32 import lru_cache + from pipenv.vendor.backports.functools_lru_cache import lru_cache else: from functools import lru_cache diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 8812d2ae..741ae319 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -21,13 +21,13 @@ pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.3 +pythonfinder==1.1.3.post1 requests==2.20.0 chardet==3.0.4 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.1.10 +requirementslib==1.2.0 attrs==18.2.0 distlib==0.2.8 packaging==18.0 @@ -41,7 +41,7 @@ semver==2.8.1 shutilwhich==1.1.0 toml==0.10.0 cached-property==1.4.3 -vistir==0.1.7 +vistir==0.2.0 pip-shims==0.3.1 ptyprocess==0.6.0 enum34==1.1.6 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index 881985d2..c8b253b8 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -11,11 +11,11 @@ from .contextmanagers import ( spinner, ) from .misc import load_path, partialclass, run, shell_escape -from .path import mkdir_p, rmtree, create_tracked_tempdir +from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile from .spin import VistirSpinner, create_spinner -__version__ = '0.1.8' +__version__ = '0.2.0' __all__ = [ @@ -36,5 +36,6 @@ __all__ = [ "spinner", "VistirSpinner", "create_spinner", - "create_tracked_tempdir" + "create_tracked_tempdir", + "create_tracked_tempfile", ] diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index 43470a6e..7b8066ee 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py @@ -175,6 +175,7 @@ def NamedTemporaryFile( prefix=None, dir=None, delete=True, + wrapper_class_override=None ): """Create and return a temporary file. Arguments: @@ -203,7 +204,10 @@ def NamedTemporaryFile( file = io.open( fd, mode, buffering=buffering, newline=newline, encoding=encoding ) - return _TemporaryFileWrapper(file, name, delete) + if wrapper_class_override is not None: + return wrapper_class_override(file, name, delete) + else: + return _TemporaryFileWrapper(file, name, delete) except BaseException: os.unlink(name) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index eab87908..ec3b65cb 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -33,9 +33,10 @@ else: from pathlib2 import Path from pipenv.vendor.backports.functools_lru_cache import lru_cache +from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile if sys.version_info < (3, 3): from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - from .backports.tempfile import NamedTemporaryFile + NamedTemporaryFile = _NamedTemporaryFile else: from tempfile import NamedTemporaryFile from shutil import get_terminal_size diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 8f25e079..bcbf7541 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -231,12 +231,13 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None): @contextmanager -def open_file(link, session=None): +def open_file(link, session=None, stream=True): """ Open local or remote file for reading. :type link: pip._internal.index.Link or str :type session: requests.Session + :param bool stream: Try to stream if remote, default True :raises ValueError: If link points to a local directory. :return: a context manager to the opened file-like object """ @@ -255,11 +256,8 @@ def open_file(link, session=None): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(link)) else: - try: - local_file = io.open(local_path, "rb") - yield local_file - finally: - local_file.close() + with io.open(local_path, "rb") as local_file: + yield local_file else: # Remote URL headers = {"Accept-Encoding": "identity"} @@ -267,8 +265,14 @@ def open_file(link, session=None): from requests import Session session = Session() - response = session.get(link, headers=headers, stream=True) - try: - yield response.raw - finally: - response.close() + with session.get(link, headers=headers, stream=stream) as resp: + try: + raw = getattr(resp, "raw", None) + result = raw if raw else resp + yield result + finally: + result.close() + if raw: + conn = getattr(raw, "_connection") + if conn is not None: + conn.close() diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index e2f85985..26382419 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -49,7 +49,7 @@ def _get_logger(name=None, level="ERROR"): formatter = logging.Formatter( "%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s" ) - handler = logging.StreamHandler() + handler = logging.StreamHandler(stream=sys.stderr) handler.setFormatter(formatter) logger.addHandler(handler) return logger @@ -157,7 +157,7 @@ def _create_subprocess( raise if not block: c.stdin.close() - log_level = "DEBUG" if verbose else "WARN" + log_level = "DEBUG" if verbose else "ERROR" logger = _get_logger(cmd._parts[0], level=log_level) output = [] err = [] @@ -199,7 +199,7 @@ def _create_subprocess( display_line = "{0}...".format(stdout_line[:display_limit]) if verbose: if spinner: - spinner.write(fs_str(display_line)) + spinner.write_err(fs_str(display_line)) else: logger.debug(display_line) if spinner: diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index ce7ecee0..d580aba2 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -15,7 +15,15 @@ import six from six.moves import urllib_parse from six.moves.urllib import request as urllib_request -from .compat import Path, _fs_encoding, TemporaryDirectory, ResourceWarning +from .backports.tempfile import _TemporaryFileWrapper +from .compat import ( + _NamedTemporaryFile, + Path, + ResourceWarning, + TemporaryDirectory, + _fs_encoding, + finalize, +) __all__ = [ @@ -28,6 +36,7 @@ __all__ = [ "mkdir_p", "ensure_mkdir_p", "create_tracked_tempdir", + "create_tracked_tempfile", "path_to_url", "rmtree", "safe_expandvars", @@ -37,6 +46,10 @@ __all__ = [ ] +if os.name == "nt" and six.PY34: + warnings.filterwarnings("ignore", category=DeprecationWarning, message="The Windows bytes API has been deprecated.*") + + def unicode_path(path): # Paths are supposed to be represented as unicode here if six.PY2 and not isinstance(path, six.text_type): @@ -52,9 +65,10 @@ def native_path(path): # once again thank you django... # https://github.com/django/django/blob/fc6b90b/django/utils/_os.py -if six.PY3 or os.name == 'nt': +if six.PY3 or os.name == "nt": abspathu = os.path.abspath else: + def abspathu(path): """ Version of os.path.abspath that uses the unicode representation @@ -74,6 +88,7 @@ def normalize_drive(path): always converted to uppercase because it seems to be preferred. """ from .misc import to_text + if os.name != "nt" or not isinstance(path, six.string_types): return path @@ -110,6 +125,7 @@ def url_to_path(url): Follows logic taken from pip's equivalent function """ from .misc import to_bytes + assert is_file_url(url), "Only file: urls can be converted to local paths" _, netloc, path, _, _ = urllib_parse.urlsplit(url) # Netlocs are UNC paths @@ -123,6 +139,7 @@ def url_to_path(url): def is_valid_url(url): """Checks if a given string is an url""" from .misc import to_text + if not url: return url pieces = urllib_parse.urlparse(to_text(url)) @@ -132,6 +149,7 @@ def is_valid_url(url): def is_file_url(url): """Returns true if the given url is a file url""" from .misc import to_text + if not url: return False if not isinstance(url, six.string_types): @@ -149,6 +167,7 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ from .misc import to_bytes + fn = to_bytes(fn, encoding="utf-8") if os.path.exists(fn): return bool(os.stat(fn).st_mode & stat.S_IREAD) and not os.access(fn, os.W_OK) @@ -164,6 +183,7 @@ def mkdir_p(newdir, mode=0o777): """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ from .misc import to_bytes, to_text + newdir = to_bytes(newdir, "utf-8") if os.path.exists(newdir): if not os.path.isdir(newdir): @@ -176,7 +196,9 @@ def mkdir_p(newdir, mode=0o777): head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) # Make sure the tail doesn't point to the asame place as the head curdir = to_bytes(".", encoding="utf-8") - tail_and_head_match = os.path.relpath(tail, start=os.path.basename(head)) == curdir + tail_and_head_match = ( + os.path.relpath(tail, start=os.path.basename(head)) == curdir + ) if tail and not tail_and_head_match and not os.path.isdir(newdir): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): @@ -191,6 +213,7 @@ def mkdir_p(newdir, mode=0o777): def ensure_mkdir_p(mode=0o777): """Decorator to ensure `mkdir_p` is called to the function's return value. """ + def decorator(f): @functools.wraps(f) @@ -224,6 +247,19 @@ def create_tracked_tempdir(*args, **kwargs): return tempdir.name +def create_tracked_tempfile(*args, **kwargs): + """Create a tracked temporary file. + + This uses the `NamedTemporaryFile` construct, but does not remove the file + until the interpreter exits. + + The return value is the file object. + """ + + kwargs["wrapper_class_override"] = _TrackedTempfileWrapper + return _NamedTemporaryFile(*args, **kwargs) + + def set_write_bit(fn): """Set read-write permissions for the current user on the target path. Fail silently if the path doesn't exist. @@ -232,6 +268,7 @@ def set_write_bit(fn): """ from .misc import to_bytes, locale_encoding + fn = to_bytes(fn, encoding=locale_encoding) if not os.path.exists(fn): return @@ -253,6 +290,7 @@ def rmtree(directory, ignore_errors=False): """ from .misc import locale_encoding, to_bytes + directory = to_bytes(directory, encoding=locale_encoding) try: shutil.rmtree( @@ -278,10 +316,12 @@ def handle_remove_readonly(func, path, exc): This function will call check :func:`is_readonly_path` before attempting to call :func:`set_write_bit` on the target path and try again. """ + # Check for read-only attribute if six.PY2: from .compat import ResourceWarning from .misc import to_bytes + PERM_ERRORS = (errno.EACCES, errno.EPERM) default_warning_message = ( "Unable to remove file due to permissions restriction: {!r}" @@ -418,3 +458,28 @@ def safe_expandvars(value): if isinstance(value, six.string_types): return os.path.expandvars(value) return value + + +class _TrackedTempfileWrapper(_TemporaryFileWrapper): + def __init__(self, *args, **kwargs): + super(_TrackedTempfileWrapper, self).__init__(*args, **kwargs) + self._finalizer = finalize(self, self.cleanup) + + @classmethod + def _cleanup(cls, fileobj): + try: + fileobj.close() + finally: + os.unlink(fileobj.name) + + def cleanup(self): + if self._finalizer.detach(): + try: + self.close() + finally: + os.unlink(self.name) + else: + try: + self.close() + except OSError: + pass diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index d2cddd79..e4b4ba66 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -3,7 +3,7 @@ import os import signal import sys -from .termcolors import colored +from .termcolors import colored, COLORS from .compat import fs_str import cursor @@ -15,6 +15,7 @@ except ImportError: Spinners = None else: from yaspin.spinners import Spinners + from yaspin.constants import COLOR_MAP handler = None if yaspin and os.name == "nt": @@ -41,6 +42,16 @@ class DummySpinner(object): self.write_err(traceback) return False + def __getattr__(self, k): + try: + retval = super(DummySpinner, self).__getattribute__(k) + except AttributeError: + if k in COLOR_MAP.keys() or k.upper() in COLORS: + return self + raise + else: + return retval + def fail(self, exitcode=1, text=None): if text: self.write_err(text) @@ -125,6 +136,42 @@ class VistirSpinner(base_obj): ) return fn + def _register_signal_handlers(self): + # SIGKILL cannot be caught or ignored, and the receiving + # process cannot perform any clean-up upon receiving this + # signal. + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass + + for sig, sig_handler in self._sigmap.items(): + # A handler for a particular signal, once set, remains + # installed until it is explicitly reset. Store default + # signal handlers for subsequent reset at cleanup phase. + dfl_handler = signal.getsignal(sig) + self._dfl_sigmap[sig] = dfl_handler + + # ``signal.SIG_DFL`` and ``signal.SIG_IGN`` are also valid + # signal handlers and are not callables. + if callable(sig_handler): + # ``signal.signal`` accepts handler function which is + # called with two arguments: signal number and the + # interrupted stack frame. ``functools.partial`` solves + # the problem of passing spinner instance into the handler + # function. + sig_handler = functools.partial(sig_handler, spinner=self) + + signal.signal(sig, sig_handler) + + def _reset_signal_handlers(self): + for sig, sig_handler in self._dfl_sigmap.items(): + signal.signal(sig, sig_handler) + @staticmethod def _hide_cursor(): cursor.hide() diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py index 6f3ad32c..b8ccda8e 100644 --- a/pipenv/vendor/vistir/termcolors.py +++ b/pipenv/vendor/vistir/termcolors.py @@ -79,6 +79,7 @@ def colored(text, color=None, on_color=None, attrs=None): style = "BRIGHT" attrs.remove('bold') if color is not None: + color = color.upper() text = text = "%s%s%s%s%s" % ( getattr(colorama.Fore, color), getattr(colorama.Style, style), @@ -88,8 +89,9 @@ def colored(text, color=None, on_color=None, attrs=None): ) if on_color is not None: + on_color = on_color.upper() text = "%s%s%s%s" % ( - getattr(colorama.Back, color), + getattr(colorama.Back, on_color), text, colorama.Back.RESET, colorama.Style.NORMAL, diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index d01fb98e..06b8b621 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -16,6 +16,9 @@ import sys import threading import time +import colorama +import cursor + from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS @@ -23,6 +26,9 @@ from .helpers import to_unicode from .termcolor import colored +colorama.init() + + class Yaspin(object): """Implements a context manager that spawns a thread to write spinner frames into a tty (stdout) during @@ -369,11 +375,14 @@ class Yaspin(object): # SIGKILL cannot be caught or ignored, and the receiving # process cannot perform any clean-up upon receiving this # signal. - if signal.SIGKILL in self._sigmap.keys(): - raise ValueError( - "Trying to set handler for SIGKILL signal. " - "SIGKILL cannot be cought or ignored in POSIX systems." - ) + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass for sig, sig_handler in iteritems(self._sigmap): # A handler for a particular signal, once set, remains @@ -521,14 +530,12 @@ class Yaspin(object): @staticmethod def _hide_cursor(): - sys.stdout.write("\033[?25l") - sys.stdout.flush() + cursor.hide() @staticmethod def _show_cursor(): - sys.stdout.write("\033[?25h") - sys.stdout.flush() + cursor.show() @staticmethod def _clear_line(): - sys.stdout.write("\033[K") + sys.stdout.write(chr(27) + "[K") diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 3198c2d4..38ec032f 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -34,6 +34,7 @@ PY2_DOWNLOAD = ['enum34',] # from time to time, remove the no longer needed ones HARDCODED_LICENSE_URLS = { 'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE', + 'cursor': 'https://raw.githubusercontent.com/GijsTimmers/cursor/master/LICENSE', 'delegator.py': 'https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE', 'click-didyoumean': 'https://raw.githubusercontent.com/click-contrib/click-didyoumean/master/LICENSE', 'click-completion': 'https://raw.githubusercontent.com/click-contrib/click-completion/master/LICENSE', @@ -70,6 +71,7 @@ PATCHED_RENAMES = { LIBRARY_RENAMES = { 'pip': 'pipenv.patched.notpip', + "functools32": "pipenv.vendor.backports.functools_lru_cache", 'enum34': 'enum', } diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch index ac63825c..175efaa1 100644 --- a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -1,8 +1,26 @@ diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py -index 0c140cad..3ffb2e31 100644 +index d15aeb97..56d12458 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py -@@ -178,6 +178,7 @@ class Command(object): +@@ -7,6 +7,8 @@ import locale + import errno + + from pexpect.popen_spawn import PopenSpawn ++import pexpect ++pexpect.EOF.__module__ = "pexpect.exceptions" + + # Include `unicode` in STR_TYPES for Python 2.X + try: +@@ -110,7 +112,7 @@ class Command(object): + if self.subprocess.before: + result += self.subprocess.before + +- if self.subprocess.after: ++ if self.subprocess.after and self.subprocess.after is not pexpect.EOF: + result += self.subprocess.after + + result += self.subprocess.read() +@@ -178,6 +180,7 @@ class Command(object): # Use subprocess. if self.blocking: popen_kwargs = self._default_popen_kwargs.copy() @@ -10,11 +28,21 @@ index 0c140cad..3ffb2e31 100644 popen_kwargs["universal_newlines"] = not binary if cwd: popen_kwargs["cwd"] = cwd -@@ -233,18 +234,23 @@ class Command(object): - def block(self): +@@ -205,7 +208,10 @@ class Command(object): + if self.blocking: + raise RuntimeError("expect can only be used on non-blocking commands.") + +- self.subprocess.expect(pattern=pattern, timeout=timeout) ++ try: ++ self.subprocess.expect(pattern=pattern, timeout=timeout) ++ except pexpect.EOF: ++ pass + + def send(self, s, end=os.linesep, signal=False): + """Sends the given string or signal to std_in.""" +@@ -234,14 +240,25 @@ class Command(object): """Blocks until process is complete.""" if self._uses_subprocess: -- self.subprocess.stdin.close() # consume stdout and stderr - try: - stdout, stderr = self.subprocess.communicate() @@ -35,10 +63,21 @@ index 0c140cad..3ffb2e31 100644 + self.std_err.close() + self.subprocess.wait() else: - self.subprocess.sendeof() -- self.subprocess.proc.stdout.close() - self.subprocess.wait() -+ self.subprocess.proc.stdout.close() +- self.subprocess.wait() ++ self.subprocess.sendeof() ++ try: ++ self.subprocess.wait() ++ finally: ++ if self.subprocess.proc.stdout: ++ self.subprocess.proc.stdout.close() def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next +@@ -263,7 +280,6 @@ class Command(object): + c.run(block=False, cwd=cwd) + if data: + c.send(data) +- c.subprocess.sendeof() + c.block() + return c + diff --git a/tasks/vendoring/patches/vendor/vistir-imports.patch b/tasks/vendoring/patches/vendor/vistir-imports.patch index f93e7959..673efad8 100644 --- a/tasks/vendoring/patches/vendor/vistir-imports.patch +++ b/tasks/vendoring/patches/vendor/vistir-imports.patch @@ -1,25 +1,3 @@ -diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py -index 1f1b7a96..0c865fe6 100644 ---- a/pipenv/vendor/vistir/compat.py -+++ b/pipenv/vendor/vistir/compat.py -@@ -30,7 +30,7 @@ else: - from pathlib2 import Path - - if sys.version_info < (3, 3): -- from backports.shutil_get_terminal_size import get_terminal_size -+ from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - from .backports.tempfile import NamedTemporaryFile - else: - from tempfile import NamedTemporaryFile -@@ -39,7 +39,7 @@ else: - try: - from weakref import finalize - except ImportError: -- from backports.weakref import finalize -+ from pipenv.vendor.backports.weakref import finalize - - try: - from functools import partialmethod diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index 483a479a..43470a6e 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py @@ -33,3 +11,30 @@ index 483a479a..43470a6e 100644 __all__ = ["finalize", "NamedTemporaryFile"] +diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py +index 9ae33fdc..ec3b65cb 100644 +--- a/pipenv/vendor/vistir/compat.py ++++ b/pipenv/vendor/vistir/compat.py +@@ -31,11 +31,11 @@ if sys.version_info >= (3, 5): + from functools import lru_cache + else: + from pathlib2 import Path +- from backports.functools_lru_cache import lru_cache ++ from pipenv.vendor.backports.functools_lru_cache import lru_cache + + from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile + if sys.version_info < (3, 3): +- from backports.shutil_get_terminal_size import get_terminal_size ++ from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + NamedTemporaryFile = _NamedTemporaryFile + else: + from tempfile import NamedTemporaryFile +@@ -44,7 +44,7 @@ else: + try: + from weakref import finalize + except ImportError: +- from backports.weakref import finalize ++ from pipenv.vendor.backports.weakref import finalize + + try: + from functools import partialmethod diff --git a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch new file mode 100644 index 00000000..a1d27cd3 --- /dev/null +++ b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch @@ -0,0 +1,62 @@ +diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py +index d01fb98e..06b8b621 100644 +--- a/pipenv/vendor/yaspin/core.py ++++ b/pipenv/vendor/yaspin/core.py +@@ -16,6 +16,9 @@ import sys + import threading + import time + ++import colorama ++import cursor ++ + from .base_spinner import default_spinner + from .compat import PY2, basestring, builtin_str, bytes, iteritems, str + from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS +@@ -23,6 +26,9 @@ from .helpers import to_unicode + from .termcolor import colored + + ++colorama.init() ++ ++ + class Yaspin(object): + """Implements a context manager that spawns a thread + to write spinner frames into a tty (stdout) during +@@ -369,11 +375,14 @@ class Yaspin(object): + # SIGKILL cannot be caught or ignored, and the receiving + # process cannot perform any clean-up upon receiving this + # signal. +- if signal.SIGKILL in self._sigmap.keys(): +- raise ValueError( +- "Trying to set handler for SIGKILL signal. " +- "SIGKILL cannot be cought or ignored in POSIX systems." +- ) ++ try: ++ if signal.SIGKILL in self._sigmap.keys(): ++ raise ValueError( ++ "Trying to set handler for SIGKILL signal. " ++ "SIGKILL cannot be cought or ignored in POSIX systems." ++ ) ++ except AttributeError: ++ pass + + for sig, sig_handler in iteritems(self._sigmap): + # A handler for a particular signal, once set, remains +@@ -521,14 +530,12 @@ class Yaspin(object): + + @staticmethod + def _hide_cursor(): +- sys.stdout.write("\033[?25l") +- sys.stdout.flush() ++ cursor.hide() + + @staticmethod + def _show_cursor(): +- sys.stdout.write("\033[?25h") +- sys.stdout.flush() ++ cursor.show() + + @staticmethod + def _clear_line(): +- sys.stdout.write("\033[K") ++ sys.stdout.write(chr(27) + "[K") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 12f27342..7f797f99 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,5 +1,6 @@ import json import os +import sys import warnings import pytest @@ -7,13 +8,12 @@ import pytest from pipenv._compat import TemporaryDirectory, Path from pipenv.vendor import delegator from pipenv.vendor import requests -from pipenv.vendor import six from pipenv.vendor import toml from pytest_pypi.app import prepare_packages as prepare_pypi_packages +from vistir.compat import ResourceWarning -if six.PY2: - class ResourceWarning(Warning): - pass + +warnings.filterwarnings("default", category=ResourceWarning) HAS_WARNED_GITHUB = False @@ -25,8 +25,8 @@ def check_internet(): resp = requests.get('http://httpbin.org/ip', timeout=1.0) resp.raise_for_status() except Exception: - warnings.warn('Cannot connect to HTTPBin...', ResourceWarning) - warnings.warn('Will skip tests requiring Internet', ResourceWarning) + warnings.warn('Cannot connect to HTTPBin...', RuntimeWarning) + warnings.warn('Will skip tests requiring Internet', RuntimeWarning) return False return True @@ -46,10 +46,10 @@ def check_github_ssh(): global HAS_WARNED_GITHUB if not res and not HAS_WARNED_GITHUB: warnings.warn( - 'Cannot connect to GitHub via SSH', ResourceWarning + 'Cannot connect to GitHub via SSH', RuntimeWarning ) warnings.warn( - 'Will skip tests requiring SSH access to GitHub', ResourceWarning + 'Will skip tests requiring SSH access to GitHub', RuntimeWarning ) HAS_WARNED_GITHUB = True return res @@ -70,18 +70,109 @@ def pytest_runtest_setup(item): pytest.skip('requires github ssh') +@pytest.yield_fixture +def pathlib_tmpdir(request, tmpdir): + yield Path(str(tmpdir)) + tmpdir.remove(ignore_errors=True) + + +# Borrowed from pip's test runner filesystem isolation +@pytest.fixture(autouse=True) +def isolate(pathlib_tmpdir): + """ + Isolate our tests so that things like global configuration files and the + like do not affect our test results. + We use an autouse function scoped fixture because we want to ensure that + every test has it's own isolated home directory. + """ + warnings.filterwarnings("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") + + + # Create a directory to use as our home location. + home_dir = os.path.join(str(pathlib_tmpdir), "home") + os.environ["PIPENV_NOSPIN"] = "1" + os.makedirs(home_dir) + + # Create a directory to use as a fake root + fake_root = os.path.join(str(pathlib_tmpdir), "fake-root") + os.makedirs(fake_root) + + # if sys.platform == 'win32': + # # Note: this will only take effect in subprocesses... + # home_drive, home_path = os.path.splitdrive(home_dir) + # os.environ.update({ + # 'USERPROFILE': home_dir, + # 'HOMEDRIVE': home_drive, + # 'HOMEPATH': home_path, + # }) + # for env_var, sub_path in ( + # ('APPDATA', 'AppData/Roaming'), + # ('LOCALAPPDATA', 'AppData/Local'), + # ): + # path = os.path.join(home_dir, *sub_path.split('/')) + # os.environ[env_var] = path + # os.makedirs(path) + # else: + # # Set our home directory to our temporary directory, this should force + # # all of our relative configuration files to be read from here instead + # # of the user's actual $HOME directory. + # os.environ["HOME"] = home_dir + # # Isolate ourselves from XDG directories + # os.environ["XDG_DATA_HOME"] = os.path.join(home_dir, ".local", "share") + # os.environ["XDG_CONFIG_HOME"] = os.path.join(home_dir, ".config") + # os.environ["XDG_CACHE_HOME"] = os.path.join(home_dir, ".cache") + # os.environ["XDG_RUNTIME_DIR"] = os.path.join(home_dir, ".runtime") + # os.environ["XDG_DATA_DIRS"] = ":".join([ + # os.path.join(fake_root, "usr", "local", "share"), + # os.path.join(fake_root, "usr", "share"), + # ]) + # os.environ["XDG_CONFIG_DIRS"] = os.path.join(fake_root, "etc", "xdg") + + # Configure git, because without an author name/email git will complain + # and cause test failures. + os.environ["GIT_CONFIG_NOSYSTEM"] = "1" + os.environ["GIT_AUTHOR_NAME"] = "pipenv" + os.environ["GIT_AUTHOR_EMAIL"] = "pipenv@pipenv.org" + + # We want to disable the version check from running in the tests + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true" + workon_home = os.path.join(home_dir, ".virtualenvs") + os.makedirs(workon_home) + os.environ["WORKON_HOME"] = workon_home + project_dir = os.path.join(home_dir, "pipenv_project") + os.makedirs(project_dir) + os.environ["PIPENV_PROJECT_DIR"] = project_dir + os.environ["CI"] = "1" + + # Make sure tests don't share a requirements tracker. + os.environ.pop('PIP_REQ_TRACKER', None) + + # FIXME: Windows... + os.makedirs(os.path.join(home_dir, ".config", "git")) + with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: + fp.write( + b"[user]\n\tname = pipenv\n\temail = pipenv@pipenv.org\n" + ) + + class _PipenvInstance(object): """An instance of a Pipenv Project...""" - def __init__(self, pypi=None, pipfile=True, chdir=False): + def __init__(self, pypi=None, pipfile=True, chdir=False, path=None): self.pypi = pypi self.original_umask = os.umask(0o007) self.original_dir = os.path.abspath(os.curdir) - self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') - path = Path(self._path.name) - try: - self.path = str(path.resolve()) - except OSError: - self.path = str(path.absolute()) + path = os.environ.get("PIPENV_PROJECT_DIR", None) + if not path: + self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + path = Path(self._path.name) + try: + self.path = str(path.resolve()) + except OSError: + self.path = str(path.absolute()) + else: + self._path = None + self.path = path # set file creation perms self.pipfile_path = None self.chdir = chdir @@ -101,6 +192,7 @@ class _PipenvInstance(object): os.environ['PIPENV_DONT_USE_PYENV'] = '1' os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' os.environ['PIPENV_VENV_IN_PROJECT'] = '1' + os.environ['PIPENV_NOSPIN'] = '1' if self.chdir: os.chdir(self.path) return self @@ -110,13 +202,13 @@ class _PipenvInstance(object): if self.chdir: os.chdir(self.original_dir) self.path = None - try: - self._path.cleanup() - except OSError as e: - _warn_msg = warn_msg.format(e) - warnings.warn(_warn_msg, ResourceWarning) - finally: - os.umask(self.original_umask) + if self._path: + try: + self._path.cleanup() + except OSError as e: + _warn_msg = warn_msg.format(e) + warnings.warn(_warn_msg, ResourceWarning) + os.umask(self.original_umask) def pipenv(self, cmd, block=True): if self.pipfile_path: @@ -162,7 +254,7 @@ class _PipenvInstance(object): @pytest.fixture() def PipenvInstance(): - return _PipenvInstance + yield _PipenvInstance @pytest.fixture(scope='module') diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 1f1719d0..7743804d 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -37,9 +37,9 @@ flask = "==0.12.2" """.strip() f.write(contents) - req_list = ("requests==2.14.0") + req_list = ("requests==2.14.0",) - dev_req_list = ("flask==0.12.2") + dev_req_list = ("flask==0.12.2",) c = p.pipenv('lock -r') d = p.pipenv('lock -r -d') diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 98c85602..40977ede 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -4,6 +4,7 @@ import pytest from mock import patch, Mock from first import first import pipenv.utils +import pythonfinder.utils # Pipfile format <-> requirements.txt format. @@ -215,13 +216,13 @@ class TestUtils: ), ], ) - @patch("delegator.run") + # @patch(".vendor.pythonfinder.utils.get_python_version") def test_python_version_output_variants( - self, mocked_delegator, version_output, version + self, monkeypatch, version_output, version ): - run_ret = Mock() - run_ret.out = version_output - mocked_delegator.return_value = run_ret + def mock_version(path): + return version_output.split()[1] + monkeypatch.setattr("pipenv.vendor.pythonfinder.utils.get_python_version", mock_version) assert pipenv.utils.python_version("some/path") == version @pytest.mark.utils From 6294c57070a95887199f1ce21afa09c2cc1ada22 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 25 Oct 2018 01:23:41 -0400 Subject: [PATCH 015/218] Update pythonfinder Signed-off-by: Dan Ryan More debugging info Signed-off-by: Dan Ryan Fix stdout write bugs Signed-off-by: Dan Ryan Fix spinner invocations Signed-off-by: Dan Ryan Create missing directory Signed-off-by: Dan Ryan Filesystem encode envvars Signed-off-by: Dan Ryan convert envvars to strings Signed-off-by: Dan Ryan Update encodings Signed-off-by: Dan Ryan Update vistir to init colorama Signed-off-by: Dan Ryan Update vistir version number Signed-off-by: Dan Ryan add some debugging and vsts changes Signed-off-by: Dan Ryan fix delegator mod Signed-off-by: Dan Ryan Log to stderr so we can see it Signed-off-by: Dan Ryan Log to stderr so we can see it Signed-off-by: Dan Ryan Try importing colorama... Signed-off-by: Dan Ryan change variable setting syntax Signed-off-by: Dan Ryan --- .vsts-ci/phases/run-tests.yml | 8 +- .vsts-ci/steps/run-tests.yml | 11 +- pipenv/__init__.py | 4 +- pipenv/core.py | 54 ++-- pipenv/resolver.py | 41 +-- pipenv/test_script.py | 32 ++ pipenv/utils.py | 41 ++- .../shutil_get_terminal_size/__init__.py | 8 +- .../get_terminal_size.py | 17 +- pipenv/vendor/delegator.py | 5 +- pipenv/vendor/dotenv/main.py | 7 - pipenv/vendor/passa/internals/dependencies.py | 3 +- pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/cli.py | 12 +- pipenv/vendor/pythonfinder/models/path.py | 3 +- pipenv/vendor/pythonfinder/models/pyenv.py | 2 - pipenv/vendor/pythonfinder/pythonfinder.py | 11 + pipenv/vendor/pythonfinder/utils.py | 7 +- pipenv/vendor/requirementslib/__init__.py | 8 +- .../vendor/requirementslib/models/__init__.py | 3 +- .../requirementslib/models/dependencies.py | 29 +- pipenv/vendor/requirementslib/utils.py | 5 +- pipenv/vendor/resolvelib/__init__.py | 16 + pipenv/vendor/resolvelib/providers.py | 78 +++++ pipenv/vendor/resolvelib/reporters.py | 23 ++ pipenv/vendor/resolvelib/resolvers.py | 287 ++++++++++++++++++ pipenv/vendor/resolvelib/structs.py | 67 ++++ pipenv/vendor/shutil_backports/__init__.py | 9 + .../shutil_backports/get_terminal_size.py | 100 ++++++ pipenv/vendor/vendor.txt | 6 +- pipenv/vendor/vistir/__init__.py | 5 +- pipenv/vendor/vistir/backports/tempfile.py | 6 +- pipenv/vendor/vistir/compat.py | 8 + pipenv/vendor/vistir/contextmanagers.py | 7 +- pipenv/vendor/vistir/misc.py | 61 ++-- pipenv/vendor/vistir/path.py | 9 +- pipenv/vendor/vistir/spin.py | 152 +++++++--- pipenv/vendor/vistir/termcolors.py | 52 +++- pipenv/vendor/yaspin/core.py | 27 +- .../vendor/delegator-close-filehandles.patch | 184 ++++++++++- tests/integration/conftest.py | 96 ++---- 41 files changed, 1198 insertions(+), 308 deletions(-) create mode 100644 pipenv/test_script.py create mode 100644 pipenv/vendor/resolvelib/__init__.py create mode 100644 pipenv/vendor/resolvelib/providers.py create mode 100644 pipenv/vendor/resolvelib/reporters.py create mode 100644 pipenv/vendor/resolvelib/resolvers.py create mode 100644 pipenv/vendor/resolvelib/structs.py create mode 100644 pipenv/vendor/shutil_backports/__init__.py create mode 100644 pipenv/vendor/shutil_backports/get_terminal_size.py diff --git a/.vsts-ci/phases/run-tests.yml b/.vsts-ci/phases/run-tests.yml index 81f02875..daf23e6a 100644 --- a/.vsts-ci/phases/run-tests.yml +++ b/.vsts-ci/phases/run-tests.yml @@ -11,8 +11,8 @@ steps: mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" mkdir -p "$WORKON_HOME" pip install certifi - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 + export GIT_SSL_CAINFO="$(python -m certifi)" + export LANG="C.UTF-8" export PIP_PROCESS_DEPENDENCY_LINKS="1" echo "Path: $PATH" echo "Installing Pipenv…" @@ -23,8 +23,8 @@ steps: - script: | # Fix Git SSL errors - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 + export GIT_SSL_CAINFO="$(python -m certifi)" + export LANG="C.UTF-8" export PIP_PROCESS_DEPENDENCY_LINKS="1" pipenv run pytest --junitxml=test-results.xml displayName: Run integration tests diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml index cb24b092..d893ccf9 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.vsts-ci/steps/run-tests.yml @@ -3,13 +3,14 @@ steps: # Fix Git SSL errors pip install certifi python -m certifi > cacert.txt - $env:GIT_SSL_CAINFO = $(Get-Content cacert.txt) + Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]"$(Get-Content cacert.txt)" # Shorten paths to get under MAX_PATH or else integration tests will fail # https://bugs.python.org/issue18199 - subst T: $env:TEMP - $env:TEMP = "T:\" - $env:TMP = "T:\" - D:\.venv\Scripts\pipenv run pytest -n 4 --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests + subst T: "$env:TEMP" + Write-Host "##vso[task.setvariable variable=TEMP]"T:\" + Write-Host "##vso[task.setvariable variable=TMP]"T:\" + Get-ChildItem Env + D:\.venv\Scripts\pipenv run pytest -n 4 -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests - task: PublishTestResults@2 diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 1fea44d5..8c6cec19 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -6,7 +6,6 @@ import os import sys from .__version__ import __version__ - PIPENV_ROOT = os.path.dirname(os.path.realpath(__file__)) PIPENV_VENDOR = os.sep.join([PIPENV_ROOT, "vendor"]) PIPENV_PATCHED = os.sep.join([PIPENV_ROOT, "patched"]) @@ -14,7 +13,8 @@ PIPENV_PATCHED = os.sep.join([PIPENV_ROOT, "patched"]) sys.path.insert(0, PIPENV_VENDOR) # Inject patched directory into system path. sys.path.insert(0, PIPENV_PATCHED) -os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" +from vistir.compat import fs_str +os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") # Hack to make things work better. try: if "concurrency" in sys.modules: diff --git a/pipenv/core.py b/pipenv/core.py index 82c11f1d..64f486d7 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -17,6 +17,7 @@ import warnings import six import urllib3.util as urllib3_util +from functools import partial from .cmdparse import Script from .project import Project, SourceNotFound @@ -110,7 +111,7 @@ def fix_utf8(text): @contextlib.contextmanager -def spinner(text=None, nospin=None, spinner_name=None): +def _spinner(text=None, nospin=None, spinner_name=None): if not text: text = "Running..." if not spinner_name: @@ -125,6 +126,10 @@ def spinner(text=None, nospin=None, spinner_name=None): yield sp +spinner = partial(_spinner, text="Running...", nospin=environments.PIPENV_NOSPIN, + spinner_name=environments.PIPENV_SPINNER) + + def which(command, location=None, allow_global=False): if not allow_global and location is None: if project.virtualenv_exists: @@ -322,9 +327,16 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): ) # Create a Pipfile… project.create_pipfile(python=python) - with spinner(): + with spinner(text=vistir.compat.fs_str("Importing requirements..."), + spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: # Import requirements.txt. - import_requirements() + try: + import_requirements() + except Exception: + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Failed...")) + else: + sp.ok(environments.PIPENV_SPINNER_FAIL_TEXT.format("Success!")) # Warn the user of side-effects. click.echo( u"{0}: Your {1} now contains pinned versions, if your {2} did. \n" @@ -467,14 +479,21 @@ def ensure_python(three=None, python=None): crayons.normal(fix_utf8("…"), bold=True), ) ) - with spinner(): + with spinner(text=vistir.compat.fs_str("Installing python..."), + spinner_name=environments.PIPENV_SPINNER, + nospin=environments.PIPENV_NOSPIN) as sp: try: c = pyenv.install(version) except PyenvError as e: - click.echo(fix_utf8("Something went wrong…")) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Failed...") + ) + click.echo(fix_utf8("Something went wrong…"), err=True) click.echo(crayons.blue(e.err), err=True) + else: + environments.PIPENV_SPINNER_OK_TEXT.format("Success!") # Print the results, in a beautiful blue… - click.echo(crayons.blue(c.out), err=True) + click.echo(crayons.blue(c.out), err=True) # Find the newly installed Python, hopefully. version = str(version) path_to_python = find_a_system_python(version) @@ -756,7 +775,6 @@ def do_install_dependencies( deps_list_bar = progress.bar(deps_list, width=32, label=INSTALL_LABEL if os.name != "nt" else "") - indexes = [] for dep in deps_list_bar: index = None @@ -845,13 +863,14 @@ def do_install_dependencies( # Return the subprocess' return code. sys.exit(c.return_code) else: - click.echo( - "{0} {1}{2}".format( - crayons.green("Success installing"), - crayons.green(dep.name), - crayons.green("!"), + if environments.is_verbose(): + click.echo( + "{0} {1}{2}".format( + crayons.green("Success installing"), + crayons.green(dep.as_line(include_hashes=False)), + crayons.green("!"), + ), ) - ) def convert_three_to_python(three, python): @@ -1014,9 +1033,6 @@ def do_lock( if dev_package in project.packages: dev_packages[dev_package] = project.packages[dev_package] # Resolve dev-package dependencies, with pip-tools. - pip_freeze = delegator.run( - "{0} freeze".format(escape_grouped_arguments(which_pip(allow_global=system))) - ).out sections = { "dev": { "packages": project.dev_packages, @@ -1072,7 +1088,6 @@ def do_lock( # TODO: be smarter about this. vcs_reqs, vcs_lockfile = get_vcs_deps( project, - pip_freeze, which=which, clear=clear, pre=pre, @@ -2185,6 +2200,10 @@ def _launch_windows_subprocess(script): command = system_which(script.command) options = {"universal_newlines": True} + env_strings = [ + vistir.compat.to_native_string("{0}: {1}".format(k, v)) for k, v in os.environ.items() + ] + click.echo(vistir.compat.to_native_string("\n".join(env_strings)), err=True) # Command not found, maybe this is a shell built-in? if not command: @@ -2204,6 +2223,7 @@ def _launch_windows_subprocess(script): def do_run_nt(script): + os.environ = {k: vistir.compat.fs_str(val) for k, val in os.environ.items()} p = _launch_windows_subprocess(script) p.communicate() sys.exit(p.returncode) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 2854a93a..b82088a2 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -3,7 +3,7 @@ import sys import json import logging -os.environ["PIP_PYTHON_PATH"] = sys.executable +os.environ["PIP_PYTHON_PATH"] = str(sys.executable) def _patch_path(): @@ -17,7 +17,7 @@ def _patch_path(): def get_parser(): from argparse import ArgumentParser - parser = ArgumentParser("pipenvresolver") + parser = ArgumentParser("pipenv-resolver") parser.add_argument("--pre", action="store_true", default=False) parser.add_argument("--clear", action="store_true", default=False) parser.add_argument("--verbose", "-v", action="count", default=False) @@ -41,17 +41,13 @@ def handle_parsed_args(parsed): elif parsed.verbose > 0: logging.getLogger("notpip").setLevel(logging.INFO) if "PIPENV_PACKAGES" in os.environ: - parsed.packages += os.environ["PIPENV_PACKAGES"].strip().split("\n") + parsed.packages += os.environ.get("PIPENV_PACKAGES", "").strip().split("\n") return parsed -def main(pre, clear, verbose, system, requirements_dir, packages): +def _main(pre, clear, verbose, system, requirements_dir, packages): os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) - os.environ["PIP_PYTHON_PATH"] = sys.executable - - import warnings - from pipenv.vendor.vistir.compat import ResourceWarning - warnings.filterwarnings("ignore", category=ResourceWarning) + os.environ["PIP_PYTHON_PATH"] = str(sys.executable) from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources @@ -91,22 +87,27 @@ def main(pre, clear, verbose, system, requirements_dir, packages): ) print("RESULTS:") if results: - import traceback - if isinstance(results, (Exception, traceback.types.TracebackType)): - sys.stderr.write(traceback.print_tb(results)) - sys.stderr.write(sys.exc_value()) - else: - print(json.dumps(results)) + print(json.dumps(results)) else: print(json.dumps([])) -if __name__ == "__main__": - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" +def main(): _patch_path() + import warnings + from pipenv.vendor.vistir.compat import ResourceWarning + warnings.simplefilter("ignore", category=ResourceWarning) + from pipenv.vendor import colorama + colorama.init() + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") parser = get_parser() - parsed, remaining = parser.parse_known_intermixed_args() - sys.argv = remaining + parsed, remaining = parser.parse_known_args() + # sys.argv = remaining parsed = handle_parsed_args(parsed) - main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.requirements_dir, + _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.requirements_dir, parsed.packages) + + +if __name__ == "__main__": + _patch_path() + main() diff --git a/pipenv/test_script.py b/pipenv/test_script.py new file mode 100644 index 00000000..d599ded6 --- /dev/null +++ b/pipenv/test_script.py @@ -0,0 +1,32 @@ +# -*- coding=utf-8 -*- + +import os +import sys + + +def _patch_path(): + import site + pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + pipenv_site_dir = os.path.dirname(pipenv_libdir) + site.addsitedir(pipenv_site_dir) + for _dir in ("vendor", "patched"): + sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) + + +def test_install(): + from pipenv.vendor.vistir.contextmanagers import cd + from pipenv.vendor.click.testing import CliRunner + runner = CliRunner() + with cd("/tmp/test"): + from pipenv.core import do_lock + locked = do_lock(system=False, clear=False, pre=False, keep_outdated=False, + write=True, pypi_mirror=None) + # result = runner.invoke(cli, ["lock", "--verbose"]) + # print(result.output) + # print(result.exit_code) + print(locked) + + +if __name__ == "__main__": + _patch_path() + test_install() diff --git a/pipenv/utils.py b/pipenv/utils.py index d8f2432b..7c78de34 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -258,7 +258,7 @@ def actually_resolve_deps( if sources: pip_args = prepare_pip_source_args(sources, pip_args) if environments.is_verbose(): - click_echo(crayons.blue("Using pip: {0}".format(" ".join(pip_args)))) + click_echo(crayons.blue("Using pip: {0}".format(" ".join(pip_args))), err=True) constraints_file = create_tracked_tempfile( mode="w", prefix="pipenv-", @@ -315,7 +315,7 @@ def actually_resolve_deps( click_echo( crayons.blue( "Please check your version specifier and version number. See PEP440 for more information." - ) + ), err=True ) raise RuntimeError return (resolved_tree, hashes, markers_lookup, resolver) @@ -330,12 +330,12 @@ def venv_resolve_deps( allow_global=False, pypi_mirror=None, ): - from .vendor.vistir.misc import fs_str, run - from .vendor.vistir.compat import Path + from .vendor.vistir.misc import fs_str + from .vendor.vistir.compat import Path, to_native_string from .vendor.vistir.path import create_tracked_tempdir from .cmdparse import Script from .core import spinner - from .vendor.pexpect.exceptions import EOF + from .vendor.pexpect.exceptions import EOF, TIMEOUT from .vendor import delegator from . import resolver import json @@ -364,26 +364,36 @@ def venv_resolve_deps( os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") - out = "" + out = to_native_string("") EOF.__module__ = "pexpect.exceptions" with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER, nospin=environments.PIPENV_NOSPIN) as sp: c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) - _out = u"" + _out = to_native_string("") + result = None while True: - result = c.expect(u"\n", timeout=-1) - if result is EOF or result is None: + try: + result = c.expect(u"\n", timeout=-1) + except (EOF, TIMEOUT): + pass + if result is None: break - _out = c.out - out += _out - sp.text = fs_str("Locking... {0}".format(_out[:100])) + _out = c.subprocess.before + if _out is not None: + _out = to_native_string("{0}".format(_out)) + out += _out + sp.text = to_native_string("Locking... {0}".format(_out[:100])) if environments.is_verbose(): - sp.write_err(_out.rstrip()) + if _out is not None: + sp._hide_cursor() + sp.write(_out.rstrip()) + sp._show_cursor() c.block() if c.return_code != 0: sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( "Locking Failed!" )) + click_echo(c.out.strip(), err=True) click_echo(c.err.strip(), err=True) sys.exit(c.return_code) else: @@ -394,7 +404,7 @@ def venv_resolve_deps( return json.loads(c.out.split("RESULTS:")[1].strip()) except IndexError: - click_echo(c.out.strip()) + click_echo(c.out.strip(), err=True) click_echo(c.err.strip(), err=True) raise RuntimeError("There was a problem with locking.") @@ -496,7 +506,7 @@ def resolve_deps( click_echo( "{0}: Error generating hash for {1}".format( crayons.red("Warning", bold=True), name - ) + ), err=True ) # # Collect un-collectable hashes (should work with devpi). # try: @@ -1088,7 +1098,6 @@ def extract_uri_from_vcs_dep(dep): def get_vcs_deps( project, - pip_freeze=None, which=None, clear=False, pre=False, diff --git a/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py b/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py index fa12816e..cfcbdf66 100644 --- a/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py +++ b/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py @@ -1,9 +1,11 @@ -__title__ = "shutil_backports" -__version__ = "0.1.0" +"""A backport of the get_terminal_size function from Python 3.3's shutil.""" + +__title__ = "backports.shutil_get_terminal_size" +__version__ = "1.0.0" __license__ = "MIT" __author__ = "Christopher Rosell" __copyright__ = "Copyright 2014 Christopher Rosell" __all__ = ["get_terminal_size"] -from .get_terminal_size import * +from .get_terminal_size import get_terminal_size diff --git a/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py b/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py index f1336e58..28c96da8 100644 --- a/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py +++ b/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py @@ -39,7 +39,7 @@ try: except Exception: pass - return columns, lines + return terminal_size(columns, lines) except ImportError: import fcntl @@ -52,7 +52,7 @@ except ImportError: except Exception: columns = lines = 0 - return columns, lines + return terminal_size(columns, lines) def get_terminal_size(fallback=(80, 24)): @@ -74,7 +74,7 @@ def get_terminal_size(fallback=(80, 24)): The value returned is a named tuple of type os.terminal_size. """ - # Attempt to use the environment first + # Try the environment first try: columns = int(os.environ["COLUMNS"]) except (KeyError, ValueError): @@ -88,13 +88,14 @@ def get_terminal_size(fallback=(80, 24)): # Only query if necessary if columns <= 0 or lines <= 0: try: - columns, lines = _get_terminal_size(sys.__stdout__.fileno()) + size = _get_terminal_size(sys.__stdout__.fileno()) except (NameError, OSError): - pass + size = terminal_size(*fallback) - # Use fallback as last resort - if columns <= 0 and lines <= 0: - columns, lines = fallback + if columns <= 0: + columns = size.columns + if lines <= 0: + lines = size.lines return terminal_size(columns, lines) diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py index 56d12458..cf6f91c8 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py @@ -113,7 +113,10 @@ class Command(object): result += self.subprocess.before if self.subprocess.after and self.subprocess.after is not pexpect.EOF: - result += self.subprocess.after + try: + result += self.subprocess.after + except (pexpect.EOF, pexpect.TIMEOUT): + pass result += self.subprocess.read() return result diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py index 6ba28bbb..4bf72946 100644 --- a/pipenv/vendor/dotenv/main.py +++ b/pipenv/vendor/dotenv/main.py @@ -95,13 +95,6 @@ class DotEnv(): for k, v in self.dict().items(): if k in os.environ and not override: continue - # With Python 2 on Windows, ensuree environment variables are - # system strings to avoid "TypeError: environment can only contain - # strings" in Python's subprocess module. - if sys.version_info.major < 3 and sys.platform == 'win32': - from pipenv.utils import fs_str - k = fs_str(k) - v = fs_str(v) os.environ[k] = v return True diff --git a/pipenv/vendor/passa/internals/dependencies.py b/pipenv/vendor/passa/internals/dependencies.py index 53b19b17..6ce97aad 100644 --- a/pipenv/vendor/passa/internals/dependencies.py +++ b/pipenv/vendor/passa/internals/dependencies.py @@ -154,6 +154,7 @@ def _get_dependencies_from_json(ireq, sources): return dependencies except Exception as e: print("unable to read dependencies via {0} ({1})".format(url, e)) + session.close() return @@ -215,7 +216,7 @@ def _read_requires_python(metadata): def _get_dependencies_from_pip(ireq, sources): - """Retrieves dependencies for the requirement from pipenv.patched.notpip internals. + """Retrieves dependencies for the requirement from pip internals. The current strategy is to try the followings in order, returning the first successful result. diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 91e9cabb..d800f926 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.3.post1' +__version__ = '1.1.5' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index 1757c081..6e7980fe 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -30,17 +30,23 @@ def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsup sys.exit(0) finder = Finder(ignore_unsupported=ignore_unsupported) if findall: - versions = finder.find_all_python_versions() + versions = [v for v in finder.find_all_python_versions()] if versions: click.secho("Found python at the following locations:", fg="green") for v in versions: py = v.py_version + comes_from = getattr(py, "comes_from", None) + if comes_from is not None: + comes_from_path = getattr(comes_from, "path", v.path) + else: + comes_from_path = v.path click.secho( - "{py.name!s}: {py.version!s} ({py.architecture!s}) @ {py.comes_from.path!s}".format( - py=py + "{py.name!s}: {py.version!s} ({py.architecture!s}) @ {comes_from!s}".format( + py=py, comes_from=comes_from_path ), fg="yellow", ) + sys.exit(0) else: click.secho( "ERROR: No valid python versions found! Check your path and try again.", diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index f39299a3..4a4c50da 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -27,7 +27,6 @@ from ..utils import ( path_is_known_executable, unnest, ) -from .python import PythonVersion @attr.s @@ -504,7 +503,6 @@ class PathEntry(BasePath): children = {} child_creation_args = { "is_root": False, - "py_version": python, "only_python": only_python } if not guessed_name: @@ -512,6 +510,7 @@ class PathEntry(BasePath): for pth, python in pythons.items(): pth = ensure_path(pth) children[pth.as_posix()] = PathEntry( + py_version=python, path=pth, **child_creation_args ) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 1595a963..a9d6105b 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -14,8 +14,6 @@ from vistir.compat import Path from ..utils import ( ensure_path, optional_instance_of, - get_python_version, - filter_pythons, unnest, ) from .mixins import BaseFinder, BasePath diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 19a52e0a..854cc8e7 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -4,6 +4,7 @@ import os import six import operator from .models import SystemPath +from vistir.compat import lru_cache class Finder(object): @@ -34,6 +35,14 @@ class Finder(object): self._system_path = None self._windows_finder = None + def __hash__(self): + return hash( + (self.path_prepend, self.system, self.global_search, self.ignore_unsupported) + ) + + def __eq__(self, other): + return self.__hash__() == other.__hash__() + @property def system_path(self): if not self._system_path: @@ -56,6 +65,7 @@ class Finder(object): def which(self, exe): return self.system_path.which(exe) + @lru_cache(maxsize=128) def find_python_version( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): @@ -103,6 +113,7 @@ class Finder(object): major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) + @lru_cache(maxsize=128) def find_all_python_versions( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 2f5a860d..fd5ac99d 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -49,12 +49,13 @@ def get_python_version(path): """Get python version string using subprocess from a given path.""" version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: - out, _ = vistir.misc.run(version_cmd, block=True, nospin=True) + c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, + combine_stderr=False) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) - if not out: + if not c.out: raise InvalidPythonVersion("%s is not a valid python path" % path) - return out.strip() + return c.out.strip() def optional_instance_of(cls): diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 7b4b6376..ba40e5f5 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,9 +1,13 @@ # -*- coding=utf-8 -*- -__version__ = '1.2.0' +__version__ = '1.2.1' + +import logging + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) from .models.requirements import Requirement from .models.lockfile import Lockfile from .models.pipfile import Pipfile - __all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/models/__init__.py b/pipenv/vendor/requirementslib/models/__init__.py index 8d12da45..99e519d6 100644 --- a/pipenv/vendor/requirementslib/models/__init__.py +++ b/pipenv/vendor/requirementslib/models/__init__.py @@ -2,10 +2,9 @@ from __future__ import absolute_import -__all__ = ["Requirement", "Lockfile", "Pipfile", "DependencyResolver"] +__all__ = ["Requirement", "Lockfile", "Pipfile"] from .requirements import Requirement from .lockfile import Lockfile from .pipfile import Pipfile -from .resolvers import DependencyResolver diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index d9f1b653..c3df222f 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -9,6 +9,7 @@ import attr import packaging.markers import packaging.version import requests +import warnings from first import first from packaging.utils import canonicalize_name @@ -17,7 +18,7 @@ from pip_shims import ( FormatControl, InstallRequirement, PackageFinder, RequirementPreparer, RequirementSet, RequirementTracker, Resolver, WheelCache, pip_version ) -from vistir.compat import JSONDecodeError, fs_str +from vistir.compat import JSONDecodeError, fs_str, ResourceWarning from vistir.contextmanagers import cd, temp_environ from vistir.misc import partialclass from vistir.path import create_tracked_tempdir @@ -53,6 +54,8 @@ def find_all_matches(finder, ireq, pre=False): :return: A list of matching candidates. :rtype: list[:class:`~pip._internal.index.InstallationCandidate`] """ + + candidates = clean_requires_python(finder.find_all_candidates(ireq.name)) versions = {candidate.version for candidate in candidates} allowed_versions = _get_filtered_versions(ireq, versions, pre) @@ -341,9 +344,12 @@ def get_dependencies_from_json(ireq): def gen(ireq): info = None - info = session.get( - "https://pypi.org/pypi/{0}/{1}/json".format(ireq.req.name, version) - ).json()["info"] + try: + info = session.get( + "https://pypi.org/pypi/{0}/{1}/json".format(ireq.req.name, version) + ).json()["info"] + finally: + session.close() requires_dist = info.get("requires_dist", info.get("requires")) if not requires_dist: # The API can return None for this. return @@ -605,13 +611,16 @@ def start_resolver(finder=None, wheel_cache=None): wheel_cache=wheel_cache, use_user_site=False, ) - if packaging.version.parse(pip_version) >= packaging.version.parse('18'): - with RequirementTracker() as req_tracker: - preparer = preparer(req_tracker=req_tracker) + try: + if packaging.version.parse(pip_version) >= packaging.version.parse('18'): + with RequirementTracker() as req_tracker: + preparer = preparer(req_tracker=req_tracker) + yield resolver(preparer=preparer) + else: + preparer = preparer() yield resolver(preparer=preparer) - else: - preparer = preparer() - yield resolver(preparer=preparer) + finally: + finder.session.close() def get_grouped_dependencies(constraints): diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index 085f3241..75a05ce0 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -6,6 +6,7 @@ import logging import os import six +import sys import tomlkit six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) @@ -23,8 +24,6 @@ from vistir.compat import Path from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir - - VCS_LIST = ("git", "svn", "hg", "bzr") VCS_SCHEMES = [] SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") @@ -38,7 +37,7 @@ if not VCS_SCHEMES: def setup_logger(): logger = logging.getLogger("requirementslib") loglevel = logging.DEBUG - handler = logging.StreamHandler() + handler = logging.StreamHandler(stream=sys.stderr) handler.setLevel(loglevel) logger.addHandler(handler) logger.setLevel(loglevel) diff --git a/pipenv/vendor/resolvelib/__init__.py b/pipenv/vendor/resolvelib/__init__.py new file mode 100644 index 00000000..e0e37434 --- /dev/null +++ b/pipenv/vendor/resolvelib/__init__.py @@ -0,0 +1,16 @@ +__all__ = [ + '__version__', + 'AbstractProvider', 'BaseReporter', 'Resolver', + 'NoVersionsAvailable', 'RequirementsConflicted', + 'ResolutionError', 'ResolutionImpossible', 'ResolutionTooDeep', +] + +__version__ = '0.2.2' + + +from .providers import AbstractProvider +from .reporters import BaseReporter +from .resolvers import ( + NoVersionsAvailable, RequirementsConflicted, + Resolver, ResolutionError, ResolutionImpossible, ResolutionTooDeep, +) diff --git a/pipenv/vendor/resolvelib/providers.py b/pipenv/vendor/resolvelib/providers.py new file mode 100644 index 00000000..515c0db4 --- /dev/null +++ b/pipenv/vendor/resolvelib/providers.py @@ -0,0 +1,78 @@ +class AbstractProvider(object): + """Delegate class to provide requirment interface for the resolver. + """ + def identify(self, dependency): + """Given a dependency, return an identifier for it. + + This is used in many places to identify the dependency, e.g. whether + two requirements should have their specifier parts merged, whether + two specifications would conflict with each other (because they the + same name but different versions). + """ + raise NotImplementedError + + def get_preference(self, resolution, candidates, information): + """Produce a sort key for given specification based on preference. + + The preference is defined as "I think this requirement should be + resolved first". The lower the return value is, the more preferred + this group of arguments is. + + :param resolution: Currently pinned candidate, or `None`. + :param candidates: A list of possible candidates. + :param information: A list of requirement information. + + Each information instance is a named tuple with two entries: + + * `requirement` specifies a requirement contributing to the current + candidate list + * `parent` specifies the candidate that provids (dependend on) the + requirement, or `None` to indicate a root requirement. + + The preference could depend on a various of issues, including (not + necessarily in this order): + + * Is this package pinned in the current resolution result? + * How relaxed is the requirement? Stricter ones should probably be + worked on first? (I don't know, actually.) + * How many possibilities are there to satisfy this requirement? Those + with few left should likely be worked on first, I guess? + * Are there any known conflicts for this requirement? We should + probably work on those with the most known conflicts. + + A sortable value should be returned (this will be used as the `key` + parameter of the built-in sorting function). The smaller the value is, + the more preferred this specification is (i.e. the sorting function + is called with `reverse=False`). + """ + raise NotImplementedError + + def find_matches(self, requirement): + """Find all possible candidates that satisfy a requirement. + + This should try to get candidates based on the requirement's type. + For VCS, local, and archive requirements, the one-and-only match is + returned, and for a "named" requirement, the index(es) should be + consulted to find concrete candidates for this requirement. + + The returned candidates should be sorted by reversed preference, e.g. + the latest should be LAST. This is done so list-popping can be as + efficient as possible. + """ + raise NotImplementedError + + def is_satisfied_by(self, requirement, candidate): + """Whether the given requirement can be satisfied by a candidate. + + A boolean should be retuened to indicate whether `candidate` is a + viable solution to the requirement. + """ + raise NotImplementedError + + def get_dependencies(self, candidate): + """Get dependencies of a candidate. + + This should return a collection of requirements that `candidate` + specifies as its dependencies. + """ + raise NotImplementedError diff --git a/pipenv/vendor/resolvelib/reporters.py b/pipenv/vendor/resolvelib/reporters.py new file mode 100644 index 00000000..c723031f --- /dev/null +++ b/pipenv/vendor/resolvelib/reporters.py @@ -0,0 +1,23 @@ +class BaseReporter(object): + """Delegate class to provider progress reporting for the resolver. + """ + def starting(self): + """Called before the resolution actually starts. + """ + + def starting_round(self, index): + """Called before each round of resolution starts. + + The index is zero-based. + """ + + def ending_round(self, index, state): + """Called before each round of resolution ends. + + This is NOT called if the resolution ends at this round. Use `ending` + if you want to report finalization. The index is zero-based. + """ + + def ending(self, state): + """Called before the resolution ends successfully. + """ diff --git a/pipenv/vendor/resolvelib/resolvers.py b/pipenv/vendor/resolvelib/resolvers.py new file mode 100644 index 00000000..9c69628e --- /dev/null +++ b/pipenv/vendor/resolvelib/resolvers.py @@ -0,0 +1,287 @@ +import collections + +from .structs import DirectedGraph + + +RequirementInformation = collections.namedtuple('RequirementInformation', [ + 'requirement', 'parent', +]) + + +class NoVersionsAvailable(Exception): + def __init__(self, requirement, parent): + super(NoVersionsAvailable, self).__init__() + self.requirement = requirement + self.parent = parent + + +class RequirementsConflicted(Exception): + def __init__(self, criterion): + super(RequirementsConflicted, self).__init__() + self.criterion = criterion + + +class Criterion(object): + """Internal representation of possible resolution results of a package. + + This holds two attributes: + + * `information` is a collection of `RequirementInformation` pairs. Each + pair is a requirement contributing to this criterion, and the candidate + that provides the requirement. + * `candidates` is a collection containing all possible candidates deducted + from the union of contributing requirements. It should never be empty. + """ + def __init__(self, candidates, information): + self.candidates = candidates + self.information = information + + @classmethod + def from_requirement(cls, provider, requirement, parent): + """Build an instance from a requirement. + """ + candidates = provider.find_matches(requirement) + if not candidates: + raise NoVersionsAvailable(requirement, parent) + return cls( + candidates=candidates, + information=[RequirementInformation(requirement, parent)], + ) + + def iter_requirement(self): + return (i.requirement for i in self.information) + + def iter_parent(self): + return (i.parent for i in self.information) + + def merged_with(self, provider, requirement, parent): + """Build a new instance from this and a new requirement. + """ + infos = list(self.information) + infos.append(RequirementInformation(requirement, parent)) + candidates = [ + c for c in self.candidates + if provider.is_satisfied_by(requirement, c) + ] + if not candidates: + raise RequirementsConflicted(self) + return type(self)(candidates, infos) + + +class ResolutionError(Exception): + pass + + +class ResolutionImpossible(ResolutionError): + def __init__(self, requirements): + super(ResolutionImpossible, self).__init__() + self.requirements = requirements + + +class ResolutionTooDeep(ResolutionError): + def __init__(self, round_count): + super(ResolutionTooDeep, self).__init__(round_count) + self.round_count = round_count + + +# Resolution state in a round. +State = collections.namedtuple('State', 'mapping graph') + + +class Resolution(object): + """Stateful resolution object. + + This is designed as a one-off object that holds information to kick start + the resolution process, and holds the results afterwards. + """ + def __init__(self, provider, reporter): + self._p = provider + self._r = reporter + self._criteria = {} + self._states = [] + + @property + def state(self): + try: + return self._states[-1] + except IndexError: + raise AttributeError('state') + + def _push_new_state(self): + """Push a new state into history. + + This new state will be used to hold resolution results of the next + coming round. + """ + try: + base = self._states[-1] + except IndexError: + graph = DirectedGraph() + graph.add(None) # Sentinel as root dependencies' parent. + state = State(mapping={}, graph=graph) + else: + state = State( + mapping=base.mapping.copy(), + graph=base.graph.copy(), + ) + self._states.append(state) + + def _contribute_to_criteria(self, name, requirement, parent): + try: + crit = self._criteria[name] + except KeyError: + crit = Criterion.from_requirement(self._p, requirement, parent) + else: + crit = crit.merged_with(self._p, requirement, parent) + self._criteria[name] = crit + + def _get_criterion_item_preference(self, item): + name, criterion = item + try: + pinned = self.state.mapping[name] + except (IndexError, KeyError): + pinned = None + return self._p.get_preference( + pinned, criterion.candidates, criterion.information, + ) + + def _is_current_pin_satisfying(self, name, criterion): + try: + current_pin = self.state.mapping[name] + except KeyError: + return False + return all( + self._p.is_satisfied_by(r, current_pin) + for r in criterion.iter_requirement() + ) + + def _check_pinnability(self, candidate, dependencies): + backup = self._criteria.copy() + contributed = set() + try: + for subdep in dependencies: + key = self._p.identify(subdep) + self._contribute_to_criteria(key, subdep, parent=candidate) + contributed.add(key) + except RequirementsConflicted: + self._criteria = backup + return None + return contributed + + def _pin_candidate(self, name, criterion, candidate, child_names): + try: + self.state.graph.remove(name) + except KeyError: + pass + self.state.mapping[name] = candidate + self.state.graph.add(name) + for parent in criterion.iter_parent(): + parent_name = None if parent is None else self._p.identify(parent) + try: + self.state.graph.connect(parent_name, name) + except KeyError: + # Parent is not yet pinned. Skip now; this edge will be + # connected when the parent is being pinned. + pass + for child_name in child_names: + try: + self.state.graph.connect(name, child_name) + except KeyError: + # Child is not yet pinned. Skip now; this edge will be + # connected when the child is being pinned. + pass + + def _pin_criteria(self): + criterion_names = [name for name, _ in sorted( + self._criteria.items(), + key=self._get_criterion_item_preference, + )] + for name in criterion_names: + # Any pin may modify any criterion during the loop. Criteria are + # replaced, not updated in-place, so we need to read this value + # in the loop instead of outside. (sarugaku/resolvelib#5) + criterion = self._criteria[name] + + if self._is_current_pin_satisfying(name, criterion): + # If the current pin already works, just use it. + continue + candidates = list(criterion.candidates) + while candidates: + candidate = candidates.pop() + dependencies = self._p.get_dependencies(candidate) + child_names = self._check_pinnability(candidate, dependencies) + if child_names is None: + continue + self._pin_candidate(name, criterion, candidate, child_names) + break + else: # All candidates tried, nothing works. Give up. (?) + raise ResolutionImpossible(list(criterion.iter_requirement())) + + def resolve(self, requirements, max_rounds): + if self._states: + raise RuntimeError('already resolved') + + for requirement in requirements: + try: + name = self._p.identify(requirement) + self._contribute_to_criteria(name, requirement, parent=None) + except RequirementsConflicted as e: + # If initial requirements conflict, nothing would ever work. + raise ResolutionImpossible(e.requirements + [requirement]) + + last = None + self._r.starting() + + for round_index in range(max_rounds): + self._r.starting_round(round_index) + + self._push_new_state() + self._pin_criteria() + + curr = self.state + if last is not None and len(curr.mapping) == len(last.mapping): + # Nothing new added. Done! Remove the duplicated entry. + del self._states[-1] + self._r.ending(last) + return + last = curr + + self._r.ending_round(round_index, curr) + + raise ResolutionTooDeep(max_rounds) + + +class Resolver(object): + """The thing that performs the actual resolution work. + """ + def __init__(self, provider, reporter): + self.provider = provider + self.reporter = reporter + + def resolve(self, requirements, max_rounds=20): + """Take a collection of constraints, spit out the resolution result. + + The return value is a representation to the final resolution result. It + is a tuple subclass with two public members: + + * `mapping`: A dict of resolved candidates. Each key is an identifier + of a requirement (as returned by the provider's `identify` method), + and the value is the resolved candidate. + * `graph`: A `DirectedGraph` instance representing the dependency tree. + The vertices are keys of `mapping`, and each edge represents *why* + a particular package is included. A special vertex `None` is + included to represent parents of user-supplied requirements. + + The following exceptions may be raised if a resolution cannot be found: + + * `NoVersionsAvailable`: A requirement has no available candidates. + * `ResolutionImpossible`: A resolution cannot be found for the given + combination of requirements. + * `ResolutionTooDeep`: The dependency tree is too deeply nested and + the resolver gave up. This is usually caused by a circular + dependency, but you can try to resolve this by increasing the + `max_rounds` argument. + """ + resolution = Resolution(self.provider, self.reporter) + resolution.resolve(requirements, max_rounds=max_rounds) + return resolution.state diff --git a/pipenv/vendor/resolvelib/structs.py b/pipenv/vendor/resolvelib/structs.py new file mode 100644 index 00000000..97bd0095 --- /dev/null +++ b/pipenv/vendor/resolvelib/structs.py @@ -0,0 +1,67 @@ +class DirectedGraph(object): + """A graph structure with directed edges. + """ + def __init__(self): + self._vertices = set() + self._forwards = {} # -> Set[] + self._backwards = {} # -> Set[] + + def __iter__(self): + return iter(self._vertices) + + def __len__(self): + return len(self._vertices) + + def __contains__(self, key): + return key in self._vertices + + def copy(self): + """Return a shallow copy of this graph. + """ + other = DirectedGraph() + other._vertices = set(self._vertices) + other._forwards = {k: set(v) for k, v in self._forwards.items()} + other._backwards = {k: set(v) for k, v in self._backwards.items()} + return other + + def add(self, key): + """Add a new vertex to the graph. + """ + if key in self._vertices: + raise ValueError('vertex exists') + self._vertices.add(key) + self._forwards[key] = set() + self._backwards[key] = set() + + def remove(self, key): + """Remove a vertex from the graph, disconnecting all edges from/to it. + """ + self._vertices.remove(key) + for f in self._forwards.pop(key): + self._backwards[f].remove(key) + for t in self._backwards.pop(key): + self._forwards[t].remove(key) + + def connected(self, f, t): + return f in self._backwards[t] and t in self._forwards[f] + + def connect(self, f, t): + """Connect two existing vertices. + + Nothing happens if the vertices are already connected. + """ + if t not in self._vertices: + raise KeyError(t) + self._forwards[f].add(t) + self._backwards[t].add(f) + + def iter_edges(self): + for f, children in self._forwards.items(): + for t in children: + yield f, t + + def iter_children(self, key): + return iter(self._forwards[key]) + + def iter_parents(self, key): + return iter(self._backwards[key]) diff --git a/pipenv/vendor/shutil_backports/__init__.py b/pipenv/vendor/shutil_backports/__init__.py new file mode 100644 index 00000000..fa12816e --- /dev/null +++ b/pipenv/vendor/shutil_backports/__init__.py @@ -0,0 +1,9 @@ +__title__ = "shutil_backports" +__version__ = "0.1.0" +__license__ = "MIT" +__author__ = "Christopher Rosell" +__copyright__ = "Copyright 2014 Christopher Rosell" + +__all__ = ["get_terminal_size"] + +from .get_terminal_size import * diff --git a/pipenv/vendor/shutil_backports/get_terminal_size.py b/pipenv/vendor/shutil_backports/get_terminal_size.py new file mode 100644 index 00000000..f1336e58 --- /dev/null +++ b/pipenv/vendor/shutil_backports/get_terminal_size.py @@ -0,0 +1,100 @@ +"""This is a backport of shutil.get_terminal_size from Python 3.3. + +The original implementation is in C, but here we use the ctypes and +fcntl modules to create a pure Python version of os.get_terminal_size. +""" + +import os +import struct +import sys + +from collections import namedtuple + +__all__ = ["get_terminal_size"] + + +terminal_size = namedtuple("terminal_size", "columns lines") + +try: + from ctypes import windll, create_string_buffer + + _handles = { + 0: windll.kernel32.GetStdHandle(-10), + 1: windll.kernel32.GetStdHandle(-11), + 2: windll.kernel32.GetStdHandle(-12), + } + + def _get_terminal_size(fd): + columns = lines = 0 + + try: + handle = _handles[fd] + csbi = create_string_buffer(22) + res = windll.kernel32.GetConsoleScreenBufferInfo(handle, csbi) + if res: + res = struct.unpack("hhhhHhhhhhh", csbi.raw) + left, top, right, bottom = res[5:9] + columns = right - left + 1 + lines = bottom - top + 1 + except Exception: + pass + + return columns, lines + +except ImportError: + import fcntl + import termios + + def _get_terminal_size(fd): + try: + res = fcntl.ioctl(fd, termios.TIOCGWINSZ, b"\x00" * 4) + lines, columns = struct.unpack("hh", res) + except Exception: + columns = lines = 0 + + return columns, lines + + +def get_terminal_size(fallback=(80, 24)): + """Get the size of the terminal window. + + For each of the two dimensions, the environment variable, COLUMNS + and LINES respectively, is checked. If the variable is defined and + the value is a positive integer, it is used. + + When COLUMNS or LINES is not defined, which is the common case, + the terminal connected to sys.__stdout__ is queried + by invoking os.get_terminal_size. + + If the terminal size cannot be successfully queried, either because + the system doesn't support querying, or because we are not + connected to a terminal, the value given in fallback parameter + is used. Fallback defaults to (80, 24) which is the default + size used by many terminal emulators. + + The value returned is a named tuple of type os.terminal_size. + """ + # Attempt to use the environment first + try: + columns = int(os.environ["COLUMNS"]) + except (KeyError, ValueError): + columns = 0 + + try: + lines = int(os.environ["LINES"]) + except (KeyError, ValueError): + lines = 0 + + # Only query if necessary + if columns <= 0 or lines <= 0: + try: + columns, lines = _get_terminal_size(sys.__stdout__.fileno()) + except (NameError, OSError): + pass + + # Use fallback as last resort + if columns <= 0 and lines <= 0: + columns, lines = fallback + + return terminal_size(columns, lines) + diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 741ae319..091f27d8 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -21,13 +21,13 @@ pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.3.post1 +pythonfinder==1.1.6 requests==2.20.0 chardet==3.0.4 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.2.0 +requirementslib==1.2.1 attrs==18.2.0 distlib==0.2.8 packaging==18.0 @@ -41,7 +41,7 @@ semver==2.8.1 shutilwhich==1.1.0 toml==0.10.0 cached-property==1.4.3 -vistir==0.2.0 +vistir==0.2.2 pip-shims==0.3.1 ptyprocess==0.6.0 enum34==1.1.6 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index c8b253b8..912ab0a4 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -1,7 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals -from .compat import NamedTemporaryFile, TemporaryDirectory, partialmethod +from .compat import NamedTemporaryFile, TemporaryDirectory, partialmethod, to_native_string from .contextmanagers import ( atomic_open_for_write, cd, @@ -15,7 +15,7 @@ from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfi from .spin import VistirSpinner, create_spinner -__version__ = '0.2.0' +__version__ = '0.2.2' __all__ = [ @@ -38,4 +38,5 @@ __all__ = [ "create_spinner", "create_tracked_tempdir", "create_tracked_tempfile", + "to_native_string" ] diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index 7b8066ee..fb044acf 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py @@ -194,6 +194,8 @@ def NamedTemporaryFile( flags = _bin_openflags # Setting O_TEMPORARY in the flags causes the OS to delete # the file when it is closed. This is only supported by Windows. + if not wrapper_class_override: + wrapper_class_override = _TemporaryFileWrapper if os.name == "nt" and delete: flags |= os.O_TEMPORARY if sys.version_info < (3, 5): @@ -205,7 +207,9 @@ def NamedTemporaryFile( fd, mode, buffering=buffering, newline=newline, encoding=encoding ) if wrapper_class_override is not None: - return wrapper_class_override(file, name, delete) + return type( + str("_TempFileWrapper"), (wrapper_class_override, object), {} + )(file, name, delete) else: return _TemporaryFileWrapper(file, name, delete) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index ec3b65cb..d6e8578a 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -24,6 +24,7 @@ __all__ = [ "lru_cache", "TemporaryDirectory", "NamedTemporaryFile", + "to_native_string", ] if sys.version_info >= (3, 5): @@ -142,3 +143,10 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() + + +def to_native_string(string): + from .misc import to_text, to_bytes + if six.PY2: + return to_bytes(string) + return to_text(string) diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index bcbf7541..4d8a3191 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -118,7 +118,6 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): """ from .spin import create_spinner - spinner_func = create_spinner if nospin is False: try: import yaspin @@ -128,10 +127,10 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): " pip install --upgrade vistir[spinner]" ) else: - spinner_name = None + spinner_name = "" if not start_text and nospin is False: start_text = "Running..." - with spinner_func( + with create_spinner( spinner_name=spinner_name, text=start_text, handler_map=handler_map, @@ -271,8 +270,8 @@ def open_file(link, session=None, stream=True): result = raw if raw else resp yield result finally: - result.close() if raw: conn = getattr(raw, "_connection") if conn is not None: conn.close() + result.close() diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 26382419..7d400a4c 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -15,7 +15,7 @@ from itertools import islice import six from .cmdparse import Script -from .compat import Path, fs_str, partialmethod +from .compat import Path, fs_str, partialmethod, to_native_string from .contextmanagers import spinner as spinner if os.name != "nt": @@ -145,7 +145,8 @@ def _create_subprocess( verbose=False, spinner=None, combine_stderr=False, - display_limit=200 + display_limit=200, + start_text="" ): if not env: env = {} @@ -157,13 +158,13 @@ def _create_subprocess( raise if not block: c.stdin.close() - log_level = "DEBUG" if verbose else "ERROR" - logger = _get_logger(cmd._parts[0], level=log_level) output = [] err = [] - spinner_orig_text = "" + spinner_orig_text = None if spinner: - spinner_orig_text = getattr(spinner, "text", "") + spinner_orig_text = getattr(spinner, "text", None) + if spinner_orig_text is None: + spinner_orig_text = start_text if start_text is not None else "" streams = { "stdout": c.stdout, "stderr": c.stderr @@ -185,25 +186,32 @@ def _create_subprocess( stdout_line = line if not (stdout_line or stderr_line): break - if stderr_line: + if stderr_line is not None: err.append(stderr_line) - if verbose: + err_line = fs_str("{0}".format(stderr_line)) + if verbose and err_line is not None: if spinner: - spinner.write_err(fs_str(stderr_line)) + spinner._hide_cursor() + spinner.write_err(err_line) + spinner._show_cursor() else: - logger.error(stderr_line) - if stdout_line: + sys.stderr.write(err_line) + sys.stderr.flush() + if stdout_line is not None: output.append(stdout_line) - display_line = stdout_line + display_line = fs_str("{0}".format(stdout_line)) if len(stdout_line) > display_limit: display_line = "{0}...".format(stdout_line[:display_limit]) - if verbose: + if verbose and display_line is not None: if spinner: - spinner.write_err(fs_str(display_line)) + spinner._hide_cursor() + spinner.write_err(display_line) + spinner._show_cursor() else: - logger.debug(display_line) + sys.stderr.write(display_line) + sys.stderr.flush() if spinner: - spinner.text = fs_str("{0} {1}".format(spinner_orig_text, display_line)) + spinner.text = to_native_string("{0} {1}".format(spinner_orig_text, display_line)) continue try: c.wait() @@ -214,21 +222,19 @@ def _create_subprocess( c.stderr.close() if spinner: if c.returncode > 0: - spinner.fail("Failed...cleaning up...") - else: - spinner.text = "Complete!" + spinner.fail(to_native_string("Failed...cleaning up...")) if not os.name == "nt": - spinner.ok("✔") + spinner.ok(to_native_string("✔ Complete")) else: - spinner.ok() - c.out = "\n".join(output) + spinner.ok(to_native_string("Complete")) + c.out = "\n".join(output) if output else "" c.err = "\n".join(err) if err else "" else: c.out, c.err = c.communicate() if not block: c.wait() - c.out = fs_str("{0}".format(c.out)) if c.out else fs_str("") - c.err = fs_str("{0}".format(c.err)) if c.err else fs_str("") + c.out = to_text("{0}".format(c.out)) if c.out else fs_str("") + c.err = to_text("{0}".format(c.err)) if c.err else fs_str("") if not return_object: return c.out.strip(), c.err.strip() return c @@ -287,9 +293,7 @@ def run( cmd = Script.parse(cmd) if block or not return_object: combine_stderr = False - start_text = "Running..." - if nospin: - start_text = None + start_text = "" with spinner(spinner_name=spinner_name, start_text=start_text, nospin=nospin) as sp: return _create_subprocess( cmd, @@ -299,7 +303,8 @@ def run( cwd=cwd, verbose=verbose, spinner=sp, - combine_stderr=combine_stderr + combine_stderr=combine_stderr, + start_text=start_text ) diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index d580aba2..68e6d464 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -46,7 +46,7 @@ __all__ = [ ] -if os.name == "nt" and six.PY34: +if os.name == "nt": warnings.filterwarnings("ignore", category=DeprecationWarning, message="The Windows bytes API has been deprecated.*") @@ -215,7 +215,6 @@ def ensure_mkdir_p(mode=0o777): """ def decorator(f): - @functools.wraps(f) def decorated(*args, **kwargs): path = f(*args, **kwargs) @@ -316,10 +315,8 @@ def handle_remove_readonly(func, path, exc): This function will call check :func:`is_readonly_path` before attempting to call :func:`set_write_bit` on the target path and try again. """ - # Check for read-only attribute - if six.PY2: - from .compat import ResourceWarning + from .compat import ResourceWarning from .misc import to_bytes PERM_ERRORS = (errno.EACCES, errno.EPERM) @@ -460,7 +457,7 @@ def safe_expandvars(value): return value -class _TrackedTempfileWrapper(_TemporaryFileWrapper): +class _TrackedTempfileWrapper(_TemporaryFileWrapper, object): def __init__(self, *args, **kwargs): super(_TrackedTempfileWrapper, self).__init__(*args, **kwargs) self._finalizer = finalize(self, self.cleanup) diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index e4b4ba66..57a90277 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -1,13 +1,18 @@ # -*- coding=utf-8 -*- + +import functools import os import signal import sys -from .termcolors import colored, COLORS -from .compat import fs_str - +import colorama import cursor -import functools +import six + +from .compat import to_native_string +from .termcolors import COLOR_MAP, COLORS, colored +from io import StringIO + try: import yaspin except ImportError: @@ -15,7 +20,6 @@ except ImportError: Spinners = None else: from yaspin.spinners import Spinners - from yaspin.constants import COLOR_MAP handler = None if yaspin and os.name == "nt": @@ -28,16 +32,20 @@ CLEAR_LINE = chr(27) + "[K" class DummySpinner(object): def __init__(self, text="", **kwargs): - self.text = text + colorama.init() + self.text = to_native_string(text) + self.stdout = kwargs.get("stdout", sys.stdout) + self.stderr = kwargs.get("stderr", sys.stderr) + self.out_buff = StringIO() def __enter__(self): - if self.text: - self.write(self.text) + if self.text and self.text != "None": + self.write_err(self.text) return self def __exit__(self, exc_type, exc_val, traceback): if not exc_type: - self.ok() + self.ok(text=None) else: self.write_err(traceback) return False @@ -52,25 +60,43 @@ class DummySpinner(object): else: return retval - def fail(self, exitcode=1, text=None): - if text: + def fail(self, exitcode=1, text="FAIL"): + if text and text != "None": self.write_err(text) + if self.out_buff: + self.out_buff.close() raise SystemExit(exitcode, text) - def ok(self, text=None): - if text: - self.write(self.text) + def ok(self, text="OK"): + if text and text != "None": + self.stderr.write(self.text) + if self.out_buff: + self.out_buff.close() return 0 def write(self, text=None): - if text: - line = fs_str("{0}\n".format(text)) - sys.stdout.write(line) + if text is None or isinstance(text, six.string_types) and text == "None": + pass + self.stdout.write(to_native_string("\r")) + line = to_native_string("{0}\n".format(text)) + self.stdout.write(line) + self.stdout.write(CLEAR_LINE) def write_err(self, text=None): - if text: - line = fs_str("{0}\n".format(text)) - sys.stderr.write(line) + if text is None or isinstance(text, six.string_types) and text == "None": + pass + self.stderr.write(to_native_string("\r")) + line = to_native_string("{0}\n".format(text)) + self.stderr.write(line) + self.stderr.write(CLEAR_LINE) + + @staticmethod + def _hide_cursor(): + pass + + @staticmethod + def _show_cursor(): + pass base_obj = yaspin.core.Yaspin if yaspin is not None else DummySpinner @@ -88,6 +114,7 @@ class VistirSpinner(base_obj): """ self.handler = handler + colorama.init() sigmap = {} if handler: sigmap.update({ @@ -102,30 +129,68 @@ class VistirSpinner(base_obj): if handler_map: sigmap.update(handler_map) spinner_name = kwargs.pop("spinner_name", "bouncingBar") - text = kwargs.pop("start_text", "") + " " + kwargs.pop("text", "") - if not text: - text = "Running..." + start_text = kwargs.pop("start_text", None) + _text = kwargs.pop("text", "Running...") + kwargs["text"] = start_text if start_text is not None else _text kwargs["sigmap"] = sigmap - kwargs["spinner"] = getattr(Spinners, spinner_name, Spinners.bouncingBar) + kwargs["spinner"] = getattr(Spinners, spinner_name, "") + self.stdout = kwargs.pop("stdout", sys.stdout) + self.stderr = kwargs.pop("stderr", sys.stderr) + self.out_buff = StringIO() super(VistirSpinner, self).__init__(*args, **kwargs) self.is_dummy = bool(yaspin is None) - def fail(self, exitcode=1, *args, **kwargs): - super(VistirSpinner, self).fail(**kwargs) + def ok(self, text="OK"): + """Set Ok (success) finalizer to a spinner.""" + _text = text if text else "OK" + self._freeze(_text) - def ok(self, *args, **kwargs): - super(VistirSpinner, self).ok(*args, **kwargs) + def fail(self, text="FAIL"): + """Set fail finalizer to a spinner.""" + _text = text if text else "FAIL" + self._freeze(_text) - def write(self, *args, **kwargs): - super(VistirSpinner, self).write(*args, **kwargs) + def write(self, text): + from .misc import to_text + sys.stdout.write("\r") + self.stdout.write(CLEAR_LINE) + if text is None: + text = "" + text = to_native_string("{0}\n".format(text)) + sys.stdout.write(text) + self.out_buff.write(to_text(text)) def write_err(self, text): """Write error text in the terminal without breaking the spinner.""" + from .misc import to_text - sys.stderr.write("\r") - self._clear_err() - text = fs_str("{0}\n".format(text)) - sys.stderr.write(text) + self.stderr.write("\r") + self.stderr.write(CLEAR_LINE) + if text is None: + text = "" + text = to_native_string("{0}\n".format(text)) + self.stderr.write(text) + self.out_buff.write(to_text(text)) + + def _freeze(self, final_text): + """Stop spinner, compose last frame and 'freeze' it.""" + if not final_text: + final_text = "" + text = to_native_string(final_text) + self._last_frame = self._compose_out(text, mode="last") + + # Should be stopped here, otherwise prints after + # self._freeze call will mess up the spinner + self.stop() + self.stdout.write(self._last_frame) + + def stop(self, *args, **kwargs): + if self.stderr and self.stderr != sys.stderr: + self.stderr.close() + if self.stdout and self.stdout != sys.stdout: + self.stdout.close() + self.out_buff.close() + super(VistirSpinner, self).stop(*args, **kwargs) def _compose_color_func(self): fn = functools.partial( @@ -136,6 +201,24 @@ class VistirSpinner(base_obj): ) return fn + def _compose_out(self, frame, mode=None): + # Ensure Unicode input + + frame = to_native_string(frame) + if self._text is None: + self._text = "" + text = to_native_string(self._text) + if self._color_func is not None: + frame = self._color_func(frame) + if self._side == "right": + frame, text = text, frame + # Mode + if not mode: + out = to_native_string("\r{0} {1}".format(frame, text)) + else: + out = to_native_string("{0} {1}\n".format(frame, text)) + return out + def _register_signal_handlers(self): # SIGKILL cannot be caught or ignored, and the receiving # process cannot perform any clean-up upon receiving this @@ -191,6 +274,7 @@ class VistirSpinner(base_obj): def create_spinner(*args, **kwargs): nospin = kwargs.pop("nospin", False) + use_yaspin = kwargs.pop("use_yaspin", nospin) if nospin: return DummySpinner(*args, **kwargs) return VistirSpinner(*args, **kwargs) diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py index b8ccda8e..8395d97d 100644 --- a/pipenv/vendor/vistir/termcolors.py +++ b/pipenv/vendor/vistir/termcolors.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, print_function, unicode_literals import colorama import os +from .compat import to_native_string ATTRIBUTES = dict( @@ -53,6 +54,33 @@ COLORS = dict( ) +COLOR_MAP = { + # name: type + "blink": "attrs", + "bold": "attrs", + "concealed": "attrs", + "dark": "attrs", + "reverse": "attrs", + "underline": "attrs", + "blue": "color", + "cyan": "color", + "green": "color", + "magenta": "color", + "red": "color", + "white": "color", + "yellow": "color", + "on_blue": "on_color", + "on_cyan": "on_color", + "on_green": "on_color", + "on_grey": "on_color", + "on_magenta": "on_color", + "on_red": "on_color", + "on_white": "on_color", + "on_yellow": "on_color", +} +COLOR_ATTRS = COLOR_MAP.keys() + + RESET = colorama.Style.RESET_ALL @@ -80,25 +108,25 @@ def colored(text, color=None, on_color=None, attrs=None): attrs.remove('bold') if color is not None: color = color.upper() - text = text = "%s%s%s%s%s" % ( - getattr(colorama.Fore, color), - getattr(colorama.Style, style), - text, - colorama.Fore.RESET, - colorama.Style.NORMAL, + text = to_native_string("%s%s%s%s%s") % ( + to_native_string(getattr(colorama.Fore, color)), + to_native_string(getattr(colorama.Style, style)), + to_native_string(text), + to_native_string(colorama.Fore.RESET), + to_native_string(colorama.Style.NORMAL), ) if on_color is not None: on_color = on_color.upper() - text = "%s%s%s%s" % ( - getattr(colorama.Back, on_color), - text, - colorama.Back.RESET, - colorama.Style.NORMAL, + text = to_native_string("%s%s%s%s") % ( + to_native_string(getattr(colorama.Back, on_color)), + to_native_string(text), + to_native_string(colorama.Back.RESET), + to_native_string(colorama.Style.NORMAL), ) if attrs is not None: - fmt_str = "%s[%%dm%%s%s[9m" % ( + fmt_str = to_native_string("%s[%%dm%%s%s[9m") % ( chr(27), chr(27) ) diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index 06b8b621..d01fb98e 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -16,9 +16,6 @@ import sys import threading import time -import colorama -import cursor - from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS @@ -26,9 +23,6 @@ from .helpers import to_unicode from .termcolor import colored -colorama.init() - - class Yaspin(object): """Implements a context manager that spawns a thread to write spinner frames into a tty (stdout) during @@ -375,14 +369,11 @@ class Yaspin(object): # SIGKILL cannot be caught or ignored, and the receiving # process cannot perform any clean-up upon receiving this # signal. - try: - if signal.SIGKILL in self._sigmap.keys(): - raise ValueError( - "Trying to set handler for SIGKILL signal. " - "SIGKILL cannot be cought or ignored in POSIX systems." - ) - except AttributeError: - pass + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) for sig, sig_handler in iteritems(self._sigmap): # A handler for a particular signal, once set, remains @@ -530,12 +521,14 @@ class Yaspin(object): @staticmethod def _hide_cursor(): - cursor.hide() + sys.stdout.write("\033[?25l") + sys.stdout.flush() @staticmethod def _show_cursor(): - cursor.show() + sys.stdout.write("\033[?25h") + sys.stdout.flush() @staticmethod def _clear_line(): - sys.stdout.write(chr(27) + "[K") + sys.stdout.write("\033[K") diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch index 175efaa1..ae0bf088 100644 --- a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -1,8 +1,8 @@ -diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py -index d15aeb97..56d12458 100644 ---- a/pipenv/vendor/delegator.py -+++ b/pipenv/vendor/delegator.py -@@ -7,6 +7,8 @@ import locale +diff --git a/delegator.py b/delegator.py +index 25d21f0..582f4fe 100644 +--- a/delegator.py ++++ b/delegator.py +@@ -7,15 +7,18 @@ import locale import errno from pexpect.popen_spawn import PopenSpawn @@ -11,26 +11,150 @@ index d15aeb97..56d12458 100644 # Include `unicode` in STR_TYPES for Python 2.X try: -@@ -110,7 +112,7 @@ class Command(object): + STR_TYPES = (str, unicode) + except NameError: +- STR_TYPES = (str, ) ++ STR_TYPES = (str,) + + TIMEOUT = 30 + ++ + def pid_exists(pid): + """Check whether pid exists in the current process table.""" + if pid == 0: +@@ -43,8 +46,8 @@ def pid_exists(pid): + else: + return True + +-class Command(object): + ++class Command(object): + def __init__(self, cmd, timeout=TIMEOUT): + super(Command, self).__init__() + self.cmd = cmd +@@ -56,7 +59,7 @@ class Command(object): + self.__err = None + + def __repr__(self): +- return ''.format(self.cmd) ++ return "".format(self.cmd) + + @property + def _popen_args(self): +@@ -65,27 +68,23 @@ class Command(object): + @property + def _default_popen_kwargs(self): + return { +- 'env': os.environ.copy(), +- 'stdin': subprocess.PIPE, +- 'stdout': subprocess.PIPE, +- 'stderr': subprocess.PIPE, +- 'shell': True, +- 'universal_newlines': True, +- 'bufsize': 0 ++ "env": os.environ.copy(), ++ "stdin": subprocess.PIPE, ++ "stdout": subprocess.PIPE, ++ "stderr": subprocess.PIPE, ++ "shell": True, ++ "universal_newlines": True, ++ "bufsize": 0, + } + + @property + def _default_pexpect_kwargs(self): +- encoding = 'utf-8' +- if sys.platform == 'win32': ++ encoding = "utf-8" ++ if sys.platform == "win32": + default_encoding = locale.getdefaultlocale()[1] + if default_encoding is not None: + encoding = default_encoding +- return { +- 'env': os.environ.copy(), +- 'encoding': encoding, +- 'timeout': self.timeout +- } ++ return {"env": os.environ.copy(), "encoding": encoding, "timeout": self.timeout} + + @property + def _uses_subprocess(self): +@@ -99,18 +98,25 @@ class Command(object): + def std_out(self): + return self.subprocess.stdout + ++ @property ++ def ok(self): ++ return self.return_code == 0 ++ + @property + def _pexpect_out(self): + if self.subprocess.encoding: +- result = '' ++ result = "" + else: +- result = b'' ++ result = b"" + if self.subprocess.before: result += self.subprocess.before -- if self.subprocess.after: -+ if self.subprocess.after and self.subprocess.after is not pexpect.EOF: - result += self.subprocess.after + if self.subprocess.after: +- result += self.subprocess.after ++ try: ++ result += self.subprocess.after ++ except (pexpect.EOF, pexpect.TIMEOUT): ++ pass result += self.subprocess.read() -@@ -178,6 +180,7 @@ class Command(object): + return result +@@ -148,7 +154,7 @@ class Command(object): + def pid(self): + """The process' PID.""" + # Support for pexpect's functionality. +- if hasattr(self.subprocess, 'proc'): ++ if hasattr(self.subprocess, "proc"): + return self.subprocess.proc.pid + # Standard subprocess method. + return self.subprocess.pid +@@ -177,23 +183,24 @@ class Command(object): # Use subprocess. if self.blocking: popen_kwargs = self._default_popen_kwargs.copy() +- popen_kwargs['universal_newlines'] = not binary + del popen_kwargs["stdin"] - popen_kwargs["universal_newlines"] = not binary ++ popen_kwargs["universal_newlines"] = not binary if cwd: - popen_kwargs["cwd"] = cwd -@@ -205,7 +208,10 @@ class Command(object): +- popen_kwargs['cwd'] = cwd ++ popen_kwargs["cwd"] = cwd + if env: +- popen_kwargs['env'].update(env) ++ popen_kwargs["env"].update(env) + s = subprocess.Popen(self._popen_args, **popen_kwargs) + # Otherwise, use pexpect. + else: + pexpect_kwargs = self._default_pexpect_kwargs.copy() + if binary: +- pexpect_kwargs['encoding'] = None ++ pexpect_kwargs["encoding"] = None + if cwd: +- pexpect_kwargs['cwd'] = cwd ++ pexpect_kwargs["cwd"] = cwd + if env: +- pexpect_kwargs['env'].update(env) ++ pexpect_kwargs["env"].update(env) + # Enable Python subprocesses to work with expect functionality. +- pexpect_kwargs['env']['PYTHONUNBUFFERED'] = '1' ++ pexpect_kwargs["env"]["PYTHONUNBUFFERED"] = "1" + s = PopenSpawn(self._popen_args, **pexpect_kwargs) + self.subprocess = s + self.was_run = True +@@ -202,15 +209,18 @@ class Command(object): + """Waits on the given pattern to appear in std_out""" + if self.blocking: - raise RuntimeError("expect can only be used on non-blocking commands.") +- raise RuntimeError('expect can only be used on non-blocking commands.') ++ raise RuntimeError("expect can only be used on non-blocking commands.") - self.subprocess.expect(pattern=pattern, timeout=timeout) + try: @@ -40,7 +164,14 @@ index d15aeb97..56d12458 100644 def send(self, s, end=os.linesep, signal=False): """Sends the given string or signal to std_in.""" -@@ -234,14 +240,25 @@ class Command(object): + + if self.blocking: +- raise RuntimeError('send can only be used on non-blocking commands.') ++ raise RuntimeError("send can only be used on non-blocking commands.") + + if not signal: + if self._uses_subprocess: +@@ -233,14 +243,25 @@ class Command(object): """Blocks until process is complete.""" if self._uses_subprocess: # consume stdout and stderr @@ -73,7 +204,7 @@ index d15aeb97..56d12458 100644 def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next -@@ -263,7 +280,6 @@ class Command(object): +@@ -262,7 +283,6 @@ class Command(object): c.run(block=False, cwd=cwd) if data: c.send(data) @@ -81,3 +212,24 @@ index d15aeb97..56d12458 100644 c.block() return c +@@ -273,12 +293,12 @@ def _expand_args(command): + # Prepare arguments. + if isinstance(command, STR_TYPES): + if sys.version_info[0] == 2: +- splitter = shlex.shlex(command.encode('utf-8')) ++ splitter = shlex.shlex(command.encode("utf-8")) + elif sys.version_info[0] == 3: + splitter = shlex.shlex(command) + else: +- splitter = shlex.shlex(command.encode('utf-8')) +- splitter.whitespace = '|' ++ splitter = shlex.shlex(command.encode("utf-8")) ++ splitter.whitespace = "|" + splitter.whitespace_split = True + command = [] + +@@ -319,4 +339,3 @@ def run(command, block=True, binary=False, timeout=TIMEOUT, cwd=None, env=None): + c.block() + + return c +- diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7f797f99..5f5c1930 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -10,10 +10,11 @@ from pipenv.vendor import delegator from pipenv.vendor import requests from pipenv.vendor import toml from pytest_pypi.app import prepare_packages as prepare_pypi_packages -from vistir.compat import ResourceWarning +from vistir.compat import ResourceWarning, fs_str +from vistir.path import mkdir_p -warnings.filterwarnings("default", category=ResourceWarning) +warnings.simplefilter("default", category=ResourceWarning) HAS_WARNED_GITHUB = False @@ -85,83 +86,32 @@ def isolate(pathlib_tmpdir): We use an autouse function scoped fixture because we want to ensure that every test has it's own isolated home directory. """ - warnings.filterwarnings("ignore", category=ResourceWarning) - warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") - # Create a directory to use as our home location. home_dir = os.path.join(str(pathlib_tmpdir), "home") - os.environ["PIPENV_NOSPIN"] = "1" os.makedirs(home_dir) - - # Create a directory to use as a fake root - fake_root = os.path.join(str(pathlib_tmpdir), "fake-root") - os.makedirs(fake_root) - - # if sys.platform == 'win32': - # # Note: this will only take effect in subprocesses... - # home_drive, home_path = os.path.splitdrive(home_dir) - # os.environ.update({ - # 'USERPROFILE': home_dir, - # 'HOMEDRIVE': home_drive, - # 'HOMEPATH': home_path, - # }) - # for env_var, sub_path in ( - # ('APPDATA', 'AppData/Roaming'), - # ('LOCALAPPDATA', 'AppData/Local'), - # ): - # path = os.path.join(home_dir, *sub_path.split('/')) - # os.environ[env_var] = path - # os.makedirs(path) - # else: - # # Set our home directory to our temporary directory, this should force - # # all of our relative configuration files to be read from here instead - # # of the user's actual $HOME directory. - # os.environ["HOME"] = home_dir - # # Isolate ourselves from XDG directories - # os.environ["XDG_DATA_HOME"] = os.path.join(home_dir, ".local", "share") - # os.environ["XDG_CONFIG_HOME"] = os.path.join(home_dir, ".config") - # os.environ["XDG_CACHE_HOME"] = os.path.join(home_dir, ".cache") - # os.environ["XDG_RUNTIME_DIR"] = os.path.join(home_dir, ".runtime") - # os.environ["XDG_DATA_DIRS"] = ":".join([ - # os.path.join(fake_root, "usr", "local", "share"), - # os.path.join(fake_root, "usr", "share"), - # ]) - # os.environ["XDG_CONFIG_DIRS"] = os.path.join(fake_root, "etc", "xdg") - - # Configure git, because without an author name/email git will complain - # and cause test failures. - os.environ["GIT_CONFIG_NOSYSTEM"] = "1" - os.environ["GIT_AUTHOR_NAME"] = "pipenv" - os.environ["GIT_AUTHOR_EMAIL"] = "pipenv@pipenv.org" - - # We want to disable the version check from running in the tests - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true" - workon_home = os.path.join(home_dir, ".virtualenvs") - os.makedirs(workon_home) - os.environ["WORKON_HOME"] = workon_home - project_dir = os.path.join(home_dir, "pipenv_project") - os.makedirs(project_dir) - os.environ["PIPENV_PROJECT_DIR"] = project_dir - os.environ["CI"] = "1" - - # Make sure tests don't share a requirements tracker. - os.environ.pop('PIP_REQ_TRACKER', None) - - # FIXME: Windows... - os.makedirs(os.path.join(home_dir, ".config", "git")) + mkdir_p(os.path.join(home_dir, ".config", "git")) with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: fp.write( b"[user]\n\tname = pipenv\n\temail = pipenv@pipenv.org\n" ) + os.environ["GIT_CONFIG_NOSYSTEM"] = fs_str("1") + os.environ["GIT_AUTHOR_NAME"] = fs_str("pipenv") + os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org") + mkdir_p(os.path.join(home_dir, ".virtualenvs")) + os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) class _PipenvInstance(object): """An instance of a Pipenv Project...""" - def __init__(self, pypi=None, pipfile=True, chdir=False, path=None): + def __init__(self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None): self.pypi = pypi self.original_umask = os.umask(0o007) self.original_dir = os.path.abspath(os.curdir) + os.environ["PIPENV_NOSPIN"] = fs_str("1") + os.environ["CI"] = fs_str("1") + warnings.simplefilter("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") path = os.environ.get("PIPENV_PROJECT_DIR", None) if not path: self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') @@ -178,7 +128,7 @@ class _PipenvInstance(object): self.chdir = chdir if self.pypi: - os.environ['PIPENV_TEST_INDEX'] = '{0}/simple'.format(self.pypi.url) + os.environ['PIPENV_TEST_INDEX'] = fs_str('{0}/simple'.format(self.pypi.url)) if pipfile: p_path = os.sep.join([self.path, 'Pipfile']) @@ -189,10 +139,10 @@ class _PipenvInstance(object): self.pipfile_path = p_path def __enter__(self): - os.environ['PIPENV_DONT_USE_PYENV'] = '1' - os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' - os.environ['PIPENV_VENV_IN_PROJECT'] = '1' - os.environ['PIPENV_NOSPIN'] = '1' + os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') + os.environ['PIPENV_IGNORE_VIRTUALENVS'] = fs_str('1') + os.environ['PIPENV_VENV_IN_PROJECT'] = fs_str('1') + os.environ['PIPENV_NOSPIN'] = fs_str('1') if self.chdir: os.chdir(self.path) return self @@ -212,11 +162,11 @@ class _PipenvInstance(object): def pipenv(self, cmd, block=True): if self.pipfile_path: - os.environ['PIPENV_PIPFILE'] = self.pipfile_path + os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path) # a bit of a hack to make sure the virtualenv is created with TemporaryDirectory(prefix='pipenv-', suffix='-cache') as tempdir: - os.environ['PIPENV_CACHE_DIR'] = tempdir.name + os.environ['PIPENV_CACHE_DIR'] = fs_str(tempdir.name) c = delegator.run('pipenv {0}'.format(cmd), block=block) if 'PIPENV_CACHE_DIR' in os.environ: del os.environ['PIPENV_CACHE_DIR'] @@ -261,11 +211,11 @@ def PipenvInstance(): def pip_src_dir(request): old_src_dir = os.environ.get('PIP_SRC', '') new_src_dir = TemporaryDirectory(prefix='pipenv-', suffix='-testsrc') - os.environ['PIP_SRC'] = new_src_dir.name + os.environ['PIP_SRC'] = fs_str(new_src_dir.name) def finalize(): new_src_dir.cleanup() - os.environ['PIP_SRC'] = old_src_dir + os.environ['PIP_SRC'] = fs_str(old_src_dir) request.addfinalizer(finalize) return request From 0135ab65718ba9b3e5af7781a1836e318c994e43 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 26 Oct 2018 18:50:27 -0400 Subject: [PATCH 016/218] Fix syntax issues in vsts Signed-off-by: Dan Ryan Fix syntax issues in vsts Signed-off-by: Dan Ryan --- .vsts-ci/steps/run-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml index d893ccf9..b7053bae 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.vsts-ci/steps/run-tests.yml @@ -3,13 +3,13 @@ steps: # Fix Git SSL errors pip install certifi python -m certifi > cacert.txt - Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]"$(Get-Content cacert.txt)" + Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" # Shorten paths to get under MAX_PATH or else integration tests will fail # https://bugs.python.org/issue18199 subst T: "$env:TEMP" - Write-Host "##vso[task.setvariable variable=TEMP]"T:\" - Write-Host "##vso[task.setvariable variable=TMP]"T:\" - Get-ChildItem Env + Write-Host "##vso[task.setvariable variable=TEMP]T:\" + Write-Host "##vso[task.setvariable variable=TMP]T:\" + Get-ChildItem Env: D:\.venv\Scripts\pipenv run pytest -n 4 -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests From 2803538701ec23d47d2aea85beacc9c8d6581fd8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 26 Oct 2018 19:06:39 -0400 Subject: [PATCH 017/218] Set environment before calling which Signed-off-by: Dan Ryan --- .vsts-ci/phases/run-tests.yml | 2 +- .vsts-ci/steps/run-tests.yml | 2 ++ pipenv/core.py | 5 ++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.vsts-ci/phases/run-tests.yml b/.vsts-ci/phases/run-tests.yml index daf23e6a..fe35ad8d 100644 --- a/.vsts-ci/phases/run-tests.yml +++ b/.vsts-ci/phases/run-tests.yml @@ -14,7 +14,7 @@ steps: export GIT_SSL_CAINFO="$(python -m certifi)" export LANG="C.UTF-8" export PIP_PROCESS_DEPENDENCY_LINKS="1" - echo "Path: $PATH" + echo "Path $PATH" echo "Installing Pipenv…" pip install -e "$(pwd)" --upgrade pipenv install --deploy --dev diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml index b7053bae..c7aef0df 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.vsts-ci/steps/run-tests.yml @@ -8,7 +8,9 @@ steps: # https://bugs.python.org/issue18199 subst T: "$env:TEMP" Write-Host "##vso[task.setvariable variable=TEMP]T:\" + $env:TEMP='T:\' Write-Host "##vso[task.setvariable variable=TMP]T:\" + $env:TEMP='T:\' Get-ChildItem Env: D:\.venv\Scripts\pipenv run pytest -n 4 -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests diff --git a/pipenv/core.py b/pipenv/core.py index 64f486d7..b776f736 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1520,6 +1520,10 @@ def which_pip(allow_global=False): def system_which(command, mult=False): """Emulates the system's which. Returns None if not found.""" _which = "which -a" if not os.name == "nt" else "where" + os.environ = { + vistir.compat.fs_str(k): vistir.compat.fs_str(val) + for k, val in os.environ.items() + } c = delegator.run("{0} {1}".format(_which, command)) try: # Which Not found… @@ -2223,7 +2227,6 @@ def _launch_windows_subprocess(script): def do_run_nt(script): - os.environ = {k: vistir.compat.fs_str(val) for k, val in os.environ.items()} p = _launch_windows_subprocess(script) p.communicate() sys.exit(p.returncode) From b8db36ebdf2733e13282f6b3d0eaa4b045a90258 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 26 Oct 2018 19:10:08 -0400 Subject: [PATCH 018/218] Fix error handling with fallback in `which` implementation Signed-off-by: Dan Ryan --- pipenv/core.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index b776f736..bb415317 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1524,22 +1524,30 @@ def system_which(command, mult=False): vistir.compat.fs_str(k): vistir.compat.fs_str(val) for k, val in os.environ.items() } - c = delegator.run("{0} {1}".format(_which, command)) try: - # Which Not found… - if c.return_code == 127: - click.echo( - "{}: the {} system utility is required for Pipenv to find Python installations properly." - "\n Please install it.".format( - crayons.red("Warning", bold=True), crayons.red(_which) - ), - err=True, - ) - assert c.return_code == 0 - except AssertionError: - return None if not mult else [] - - result = c.out.strip() or c.err.strip() + c = delegator.run("{0} {1}".format(_which, command)) + try: + # Which Not found… + if c.return_code == 127: + click.echo( + "{}: the {} system utility is required for Pipenv to find Python installations properly." + "\n Please install it.".format( + crayons.red("Warning", bold=True), crayons.red(_which) + ), + err=True, + ) + assert c.return_code == 0 + except AssertionError: + return None if not mult else[] + except TypeError: + from .vendor.pythonfinder import Finder + finder = Finder() + result = finder.which(command) + if result: + return result.path.as_posix() + return + else: + result = c.out.strip() or c.err.strip() if mult: return result.split("\n") From 0cefb5ed9bd89dc89964faeb918f94ae5df4b218 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 26 Oct 2018 20:47:06 -0400 Subject: [PATCH 019/218] Finish updating vendored deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dan Ryan Update delegator patch Signed-off-by: Dan Ryan Update patches for pip Signed-off-by: Dan Ryan Update vendored deps Signed-off-by: Dan Ryan Update patches Signed-off-by: Dan Ryan Fix imports to use pip shims instead of direct imports Signed-off-by: Dan Ryan Update vendoring scripts and pip shims Signed-off-by: Dan Ryan Log to stdout in real time during verbose logging Signed-off-by: Dan Ryan Don’t log environment Fix unicode decoding issues Signed-off-by: Dan Ryan Only set buffers on ttys Signed-off-by: Dan Ryan Fix typo Signed-off-by: Dan Ryan Use default encodings Signed-off-by: Dan Ryan Fix encodings and run only failing tests Signed-off-by: Dan Ryan --- .vsts-ci/steps/run-tests.yml | 4 +- pipenv/__init__.py | 20 +- pipenv/_compat.py | 76 +++ pipenv/core.py | 19 +- pipenv/patched/notpip/__init__.py | 2 +- pipenv/patched/notpip/_internal/__init__.py | 244 +------- pipenv/patched/notpip/_internal/build_env.py | 16 + pipenv/patched/notpip/_internal/cache.py | 12 +- .../patched/notpip/_internal/cli/__init__.py | 4 + .../notpip/_internal/cli/autocompletion.py | 152 +++++ .../{basecommand.py => cli/base_command.py} | 32 +- .../notpip/_internal/{ => cli}/cmdoptions.py | 117 +++- .../notpip/_internal/cli/main_parser.py | 96 +++ .../{baseparser.py => cli/parser.py} | 29 +- .../_internal/{ => cli}/status_codes.py | 0 .../notpip/_internal/commands/__init__.py | 2 +- .../notpip/_internal/commands/check.py | 2 +- .../notpip/_internal/commands/completion.py | 2 +- .../_internal/commands/configuration.py | 4 +- .../notpip/_internal/commands/download.py | 76 +-- .../notpip/_internal/commands/freeze.py | 8 +- .../patched/notpip/_internal/commands/hash.py | 4 +- .../patched/notpip/_internal/commands/help.py | 3 +- .../notpip/_internal/commands/install.py | 29 +- .../patched/notpip/_internal/commands/list.py | 8 +- .../notpip/_internal/commands/search.py | 6 +- .../patched/notpip/_internal/commands/show.py | 4 +- .../notpip/_internal/commands/uninstall.py | 7 +- .../notpip/_internal/commands/wheel.py | 4 +- .../patched/notpip/_internal/configuration.py | 19 +- pipenv/patched/notpip/_internal/download.py | 4 +- pipenv/patched/notpip/_internal/exceptions.py | 19 + pipenv/patched/notpip/_internal/index.py | 570 ++++++------------ pipenv/patched/notpip/_internal/locations.py | 2 +- .../notpip/_internal/models/candidate.py | 24 + .../notpip/_internal/models/format_control.py | 62 ++ .../patched/notpip/_internal/models/index.py | 26 +- .../patched/notpip/_internal/models/link.py | 141 +++++ .../notpip/_internal/operations/freeze.py | 33 +- .../notpip/_internal/operations/prepare.py | 48 +- pipenv/patched/notpip/_internal/pep425tags.py | 14 +- pipenv/patched/notpip/_internal/pyproject.py | 144 +++++ .../notpip/_internal/req/constructors.py | 298 +++++++++ .../patched/notpip/_internal/req/req_file.py | 10 +- .../notpip/_internal/req/req_install.py | 413 ++----------- .../patched/notpip/_internal/req/req_set.py | 136 +++-- .../notpip/_internal/req/req_uninstall.py | 9 +- pipenv/patched/notpip/_internal/resolve.py | 13 +- .../patched/notpip/_internal/utils/appdirs.py | 2 +- .../notpip/_internal/{ => utils}/compat.py | 13 + .../notpip/_internal/utils/filesystem.py | 2 +- .../patched/notpip/_internal/utils/logging.py | 2 +- pipenv/patched/notpip/_internal/utils/misc.py | 57 +- .../patched/notpip/_internal/utils/models.py | 40 ++ .../notpip/_internal/utils/outdated.py | 25 +- .../notpip/_internal/utils/packaging.py | 26 +- pipenv/patched/notpip/_internal/utils/ui.py | 2 +- .../patched/notpip/_internal/vcs/__init__.py | 78 +-- pipenv/patched/notpip/_internal/vcs/bazaar.py | 16 +- pipenv/patched/notpip/_internal/vcs/git.py | 117 ++-- .../patched/notpip/_internal/vcs/mercurial.py | 11 +- .../notpip/_internal/vcs/subversion.py | 103 +--- pipenv/patched/notpip/_internal/wheel.py | 24 +- .../notpip/_vendor/certifi/__init__.py | 2 +- .../notpip/_vendor/certifi/__main__.py | 2 +- .../patched/notpip/_vendor/certifi/cacert.pem | 226 ++----- .../notpip/_vendor/packaging/__about__.py | 4 +- .../notpip/_vendor/packaging/requirements.py | 8 +- .../notpip/_vendor/packaging/specifiers.py | 2 +- .../notpip/_vendor/pep517/LICENSE} | 12 +- .../patched/notpip/_vendor/pep517/__init__.py | 4 + .../notpip/_vendor/pep517/_in_process.py | 182 ++++++ pipenv/patched/notpip/_vendor/pep517/check.py | 194 ++++++ .../patched/notpip/_vendor/pep517/colorlog.py | 110 ++++ .../patched/notpip/_vendor/pep517/compat.py | 23 + .../patched/notpip/_vendor/pep517/envbuild.py | 150 +++++ .../patched/notpip/_vendor/pep517/wrappers.py | 134 ++++ .../notpip/_vendor/pkg_resources/__init__.py | 67 +- .../_vendor/pkg_resources/py31compat.py | 5 +- pipenv/patched/notpip/_vendor/pyparsing.py | 46 +- .../patched/notpip/_vendor/pytoml/parser.py | 4 +- .../notpip/_vendor/requests/__init__.py | 2 +- pipenv/patched/notpip/_vendor/vendor.txt | 11 +- pipenv/patched/patched.txt | 2 +- pipenv/patched/piptools/_compat/pip_compat.py | 88 +-- pipenv/project.py | 5 +- pipenv/resolver.py | 13 +- pipenv/utils.py | 34 +- pipenv/vendor/backports/__init__.py | 6 +- pipenv/vendor/dotenv/main.py | 7 + pipenv/vendor/modutil.LICENSE | 29 - pipenv/vendor/passa/internals/dependencies.py | 2 +- pipenv/vendor/pip_shims/__init__.py | 2 +- pipenv/vendor/pip_shims/shims.py | 4 + pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/models/path.py | 1 + pipenv/vendor/pythonfinder/models/pyenv.py | 2 + pipenv/vendor/requests/LICENSE | 2 +- pipenv/vendor/requirementslib/models/utils.py | 2 +- pipenv/vendor/shutil_backports/__init__.py | 9 - .../shutil_backports/get_terminal_size.py | 100 --- pipenv/vendor/vendor.txt | 3 +- pipenv/vendor/vendor_pip.txt | 11 +- pipenv/vendor/vistir/contextmanagers.py | 13 +- pipenv/vendor/vistir/misc.py | 33 + pipenv/vendor/vistir/spin.py | 41 +- pipenv/vendor/yaspin/core.py | 27 +- pytest.ini | 2 +- tasks/vendoring/__init__.py | 13 +- .../patched/_post-pip-update-pep425tags.patch | 16 +- tasks/vendoring/patches/patched/pip18.patch | 263 ++++---- .../vendor/delegator-close-filehandles.patch | 177 +----- .../patches/vendor/passa-close-session.patch | 12 + .../patches/vendor/vistir-spin-colorama.patch | 28 + 114 files changed, 3363 insertions(+), 2259 deletions(-) create mode 100644 pipenv/patched/notpip/_internal/cli/__init__.py create mode 100644 pipenv/patched/notpip/_internal/cli/autocompletion.py rename pipenv/patched/notpip/_internal/{basecommand.py => cli/base_command.py} (92%) rename pipenv/patched/notpip/_internal/{ => cli}/cmdoptions.py (81%) create mode 100644 pipenv/patched/notpip/_internal/cli/main_parser.py rename pipenv/patched/notpip/_internal/{baseparser.py => cli/parser.py} (88%) rename pipenv/patched/notpip/_internal/{ => cli}/status_codes.py (100%) create mode 100644 pipenv/patched/notpip/_internal/models/candidate.py create mode 100644 pipenv/patched/notpip/_internal/models/format_control.py create mode 100644 pipenv/patched/notpip/_internal/models/link.py create mode 100644 pipenv/patched/notpip/_internal/pyproject.py create mode 100644 pipenv/patched/notpip/_internal/req/constructors.py rename pipenv/patched/notpip/_internal/{ => utils}/compat.py (96%) create mode 100644 pipenv/patched/notpip/_internal/utils/models.py rename pipenv/{vendor/pathlib2.LICENSE.rst => patched/notpip/_vendor/pep517/LICENSE} (83%) create mode 100644 pipenv/patched/notpip/_vendor/pep517/__init__.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/_in_process.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/check.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/colorlog.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/compat.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/envbuild.py create mode 100644 pipenv/patched/notpip/_vendor/pep517/wrappers.py delete mode 100644 pipenv/vendor/modutil.LICENSE delete mode 100644 pipenv/vendor/shutil_backports/__init__.py delete mode 100644 pipenv/vendor/shutil_backports/get_terminal_size.py create mode 100644 tasks/vendoring/patches/vendor/passa-close-session.patch create mode 100644 tasks/vendoring/patches/vendor/vistir-spin-colorama.patch diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml index c7aef0df..4c2640ab 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.vsts-ci/steps/run-tests.yml @@ -4,6 +4,7 @@ steps: pip install certifi python -m certifi > cacert.txt Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" + $env:GIT_SSL_CAINFO="$(Get-Content cacert.txt)" # Shorten paths to get under MAX_PATH or else integration tests will fail # https://bugs.python.org/issue18199 subst T: "$env:TEMP" @@ -11,8 +12,7 @@ steps: $env:TEMP='T:\' Write-Host "##vso[task.setvariable variable=TMP]T:\" $env:TEMP='T:\' - Get-ChildItem Env: - D:\.venv\Scripts\pipenv run pytest -n 4 -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests + D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor -k 'test_get_vcs_refs or test_install_editable_git_tag' --junitxml=test-results.xml tests displayName: Run integration tests - task: PublishTestResults@2 diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 8c6cec19..6b8ddf66 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -1,9 +1,13 @@ +# -*- coding=utf-8 -*- # |~~\' |~~ # |__/||~~\|--|/~\\ / # | ||__/|__| |\/ # | + import os import sys +import warnings + from .__version__ import __version__ PIPENV_ROOT = os.path.dirname(os.path.realpath(__file__)) @@ -13,14 +17,28 @@ PIPENV_PATCHED = os.sep.join([PIPENV_ROOT, "patched"]) sys.path.insert(0, PIPENV_VENDOR) # Inject patched directory into system path. sys.path.insert(0, PIPENV_PATCHED) -from vistir.compat import fs_str + +from pipenv.vendor.urllib3.exceptions import DependencyWarning +from pipenv.vendor.vistir.compat import ResourceWarning, fs_str +warnings.filterwarnings("ignore", category=DependencyWarning) +warnings.filterwarnings("ignore", category=ResourceWarning) + +if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): + if sys.stdout.isatty() and sys.stderr.isatty(): + import io + sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') + sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") +os.environ["PIP_SHIMS_BASE_MODULE"] = fs_str("pipenv.patched.notpip") + # Hack to make things work better. try: if "concurrency" in sys.modules: del sys.modules["concurrency"] except Exception: pass + from .cli import cli from . import resolver diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 223baec1..087025a4 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -300,3 +300,79 @@ def NamedTemporaryFile( os.unlink(name) os.close(fd) raise + + +def getpreferredencoding(): + import locale + # Borrowed from Invoke + # (see https://github.com/pyinvoke/invoke/blob/93af29d/invoke/runners.py#L881) + _encoding = locale.getpreferredencoding(False) + if six.PY2 and not sys.platform == "win32": + _default_encoding = locale.getdefaultlocale()[1] + if _default_encoding is not None: + _encoding = _default_encoding + return _encoding + + +DEFAULT_ENCODING = getpreferredencoding() + + +# From https://github.com/CarlFK/veyepar/blob/5c5de47/dj/scripts/fixunicode.py +# MIT LIcensed, thanks Carl! +def force_encoding(): + try: + stdout_isatty = sys.stdout.isatty + stderr_isatty = sys.stderr.isatty + except AttributeError: + return DEFAULT_ENCODING, DEFAULT_ENCODING + else: + if not (stdout_isatty() and stderr_isatty()): + return DEFAULT_ENCODING, DEFAULT_ENCODING + stdout_encoding = sys.stdout.encoding + stderr_encoding = sys.stderr.encoding + if sys.platform == "win32" and sys.version_info >= (3, 1): + return DEFAULT_ENCODING, DEFAULT_ENCODING + if stdout_encoding.lower() != "utf-8" or stderr_encoding.lower() != "utf-8": + + from ctypes import pythonapi, py_object, c_char_p + try: + PyFile_SetEncoding = pythonapi.PyFile_SetEncoding + except AttributeError: + return DEFAULT_ENCODING, DEFAULT_ENCODING + else: + PyFile_SetEncoding.argtypes = (py_object, c_char_p) + if stdout_encoding.lower() != "utf-8": + try: + was_set = PyFile_SetEncoding(sys.stdout, "utf-8") + except OSError: + was_set = False + if not was_set: + stdout_encoding = DEFAULT_ENCODING + else: + stdout_encoding = "utf-8" + + if stderr_encoding.lower() != "utf-8": + try: + was_set = PyFile_SetEncoding(sys.stderr, "utf-8") + except OSError: + was_set = False + if not was_set: + stderr_encoding = DEFAULT_ENCODING + else: + stderr_encoding = "utf-8" + + return stdout_encoding, stderr_encoding + + +OUT_ENCODING, ERR_ENCODING = force_encoding() + + +def decode_output(output): + if not isinstance(output, six.string_types): + return output + try: + output = output.encode(DEFAULT_ENCODING) + except AttributeError: + pass + output = output.decode(DEFAULT_ENCODING) + return output diff --git a/pipenv/core.py b/pipenv/core.py index bb415317..ee1658bf 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1,4 +1,5 @@ # -*- coding=utf-8 -*- + import contextlib import logging import os @@ -105,9 +106,13 @@ UNICODE_TO_ASCII_TRANSLATION_MAP = { def fix_utf8(text): if not isinstance(text, six.string_types): return text - if six.PY2: - text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) - return u"{0}".format(text) + from ._compat import decode_output + try: + text = decode_output(text) + except UnicodeDecodeError: + if six.PY2: + text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) + return text @contextlib.contextmanager @@ -230,7 +235,7 @@ def cleanup_virtualenv(bare=True): def import_requirements(r=None, dev=False): from .patched.notpip._vendor import requests as pip_requests - from .patched.notpip._internal.req.req_file import parse_requirements + from .vendor.pip_shims.shims import parse_requirements # Parse requirements.txt file with Pip's parser. # Pip requires a `PipSession` which is a subclass of requests.Session. @@ -1754,7 +1759,7 @@ def do_install( selective_upgrade=False, ): from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM - from notpip._internal.exceptions import PipError + from .vendor.pip_shims.shims import PipError requirements_directory = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" @@ -2212,10 +2217,6 @@ def _launch_windows_subprocess(script): command = system_which(script.command) options = {"universal_newlines": True} - env_strings = [ - vistir.compat.to_native_string("{0}: {1}".format(k, v)) for k, v in os.environ.items() - ] - click.echo(vistir.compat.to_native_string("\n".join(env_strings)), err=True) # Command not found, maybe this is a shell built-in? if not command: diff --git a/pipenv/patched/notpip/__init__.py b/pipenv/patched/notpip/__init__.py index 9227d0ea..ae265fa7 100644 --- a/pipenv/patched/notpip/__init__.py +++ b/pipenv/patched/notpip/__init__.py @@ -1 +1 @@ -__version__ = "18.0" +__version__ = "18.1" diff --git a/pipenv/patched/notpip/_internal/__init__.py b/pipenv/patched/notpip/_internal/__init__.py index dcd0937e..6d223928 100644 --- a/pipenv/patched/notpip/_internal/__init__.py +++ b/pipenv/patched/notpip/_internal/__init__.py @@ -4,7 +4,6 @@ from __future__ import absolute_import import locale import logging import os -import optparse import warnings import sys @@ -38,17 +37,12 @@ else: else: securetransport.inject_into_urllib3() -from pipenv.patched.notpip import __version__ -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.exceptions import CommandError, PipError -from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions, get_prog +from pipenv.patched.notpip._internal.cli.autocompletion import autocomplete +from pipenv.patched.notpip._internal.cli.main_parser import parse_command +from pipenv.patched.notpip._internal.commands import commands_dict +from pipenv.patched.notpip._internal.exceptions import PipError from pipenv.patched.notpip._internal.utils import deprecation from pipenv.patched.notpip._internal.vcs import git, mercurial, subversion, bazaar # noqa -from pipenv.patched.notpip._internal.baseparser import ( - ConfigOptionParser, UpdatingDefaultsHelpFormatter, -) -from pipenv.patched.notpip._internal.commands import get_summaries, get_similar_commands -from pipenv.patched.notpip._internal.commands import commands_dict from pipenv.patched.notpip._vendor.urllib3.exceptions import InsecureRequestWarning logger = logging.getLogger(__name__) @@ -57,232 +51,6 @@ logger = logging.getLogger(__name__) warnings.filterwarnings("ignore", category=InsecureRequestWarning) -def autocomplete(): - """Command and option completion for the main option parser (and options) - and its subcommands (and options). - - Enable by sourcing one of the completion shell scripts (bash, zsh or fish). - """ - # Don't complete if user hasn't sourced bash_completion file. - if 'PIP_AUTO_COMPLETE' not in os.environ: - return - cwords = os.environ['COMP_WORDS'].split()[1:] - cword = int(os.environ['COMP_CWORD']) - try: - current = cwords[cword - 1] - except IndexError: - current = '' - - subcommands = [cmd for cmd, summary in get_summaries()] - options = [] - # subcommand - try: - subcommand_name = [w for w in cwords if w in subcommands][0] - except IndexError: - subcommand_name = None - - parser = create_main_parser() - # subcommand options - if subcommand_name: - # special case: 'help' subcommand has no options - if subcommand_name == 'help': - sys.exit(1) - # special case: list locally installed dists for show and uninstall - should_list_installed = ( - subcommand_name in ['show', 'uninstall'] and - not current.startswith('-') - ) - if should_list_installed: - installed = [] - lc = current.lower() - for dist in get_installed_distributions(local_only=True): - if dist.key.startswith(lc) and dist.key not in cwords[1:]: - installed.append(dist.key) - # if there are no dists installed, fall back to option completion - if installed: - for dist in installed: - print(dist) - sys.exit(1) - - subcommand = commands_dict[subcommand_name]() - - for opt in subcommand.parser.option_list_all: - if opt.help != optparse.SUPPRESS_HELP: - for opt_str in opt._long_opts + opt._short_opts: - options.append((opt_str, opt.nargs)) - - # filter out previously specified options from available options - prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] - options = [(x, v) for (x, v) in options if x not in prev_opts] - # filter options by current input - options = [(k, v) for k, v in options if k.startswith(current)] - # get completion type given cwords and available subcommand options - completion_type = get_path_completion_type( - cwords, cword, subcommand.parser.option_list_all, - ) - # get completion files and directories if ``completion_type`` is - # ````, ```` or ```` - if completion_type: - options = auto_complete_paths(current, completion_type) - options = ((opt, 0) for opt in options) - for option in options: - opt_label = option[0] - # append '=' to options which require args - if option[1] and option[0][:2] == "--": - opt_label += '=' - print(opt_label) - else: - # show main parser options only when necessary - - opts = [i.option_list for i in parser.option_groups] - opts.append(parser.option_list) - opts = (o for it in opts for o in it) - if current.startswith('-'): - for opt in opts: - if opt.help != optparse.SUPPRESS_HELP: - subcommands += opt._long_opts + opt._short_opts - else: - # get completion type given cwords and all available options - completion_type = get_path_completion_type(cwords, cword, opts) - if completion_type: - subcommands = auto_complete_paths(current, completion_type) - - print(' '.join([x for x in subcommands if x.startswith(current)])) - sys.exit(1) - - -def get_path_completion_type(cwords, cword, opts): - """Get the type of path completion (``file``, ``dir``, ``path`` or None) - - :param cwords: same as the environmental variable ``COMP_WORDS`` - :param cword: same as the environmental variable ``COMP_CWORD`` - :param opts: The available options to check - :return: path completion type (``file``, ``dir``, ``path`` or None) - """ - if cword < 2 or not cwords[cword - 2].startswith('-'): - return - for opt in opts: - if opt.help == optparse.SUPPRESS_HELP: - continue - for o in str(opt).split('/'): - if cwords[cword - 2].split('=')[0] == o: - if any(x in ('path', 'file', 'dir') - for x in opt.metavar.split('/')): - return opt.metavar - - -def auto_complete_paths(current, completion_type): - """If ``completion_type`` is ``file`` or ``path``, list all regular files - and directories starting with ``current``; otherwise only list directories - starting with ``current``. - - :param current: The word to be completed - :param completion_type: path completion type(`file`, `path` or `dir`)i - :return: A generator of regular files and/or directories - """ - directory, filename = os.path.split(current) - current_path = os.path.abspath(directory) - # Don't complete paths if they can't be accessed - if not os.access(current_path, os.R_OK): - return - filename = os.path.normcase(filename) - # list all files that start with ``filename`` - file_list = (x for x in os.listdir(current_path) - if os.path.normcase(x).startswith(filename)) - for f in file_list: - opt = os.path.join(current_path, f) - comp_file = os.path.normcase(os.path.join(directory, f)) - # complete regular files when there is not ```` after option - # complete directories when there is ````, ```` or - # ````after option - if completion_type != 'dir' and os.path.isfile(opt): - yield comp_file - elif os.path.isdir(opt): - yield os.path.join(comp_file, '') - - -def create_main_parser(): - parser_kw = { - 'usage': '\n%prog [options]', - 'add_help_option': False, - 'formatter': UpdatingDefaultsHelpFormatter(), - 'name': 'global', - 'prog': get_prog(), - } - - parser = ConfigOptionParser(**parser_kw) - parser.disable_interspersed_args() - - pip_pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - parser.version = 'pip %s from %s (python %s)' % ( - __version__, pip_pkg_dir, sys.version[:3], - ) - - # add the general options - gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) - parser.add_option_group(gen_opts) - - parser.main = True # so the help formatter knows - - # create command listing for description - command_summaries = get_summaries() - description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries] - parser.description = '\n'.join(description) - - return parser - - -def parseopts(args): - parser = create_main_parser() - - # Note: parser calls disable_interspersed_args(), so the result of this - # call is to split the initial args into the general options before the - # subcommand and everything else. - # For example: - # args: ['--timeout=5', 'install', '--user', 'INITools'] - # general_options: ['--timeout==5'] - # args_else: ['install', '--user', 'INITools'] - general_options, args_else = parser.parse_args(args) - - # --version - if general_options.version: - sys.stdout.write(parser.version) - sys.stdout.write(os.linesep) - sys.exit() - - # pip || pip help -> print_help() - if not args_else or (args_else[0] == 'help' and len(args_else) == 1): - parser.print_help() - sys.exit() - - # the subcommand name - cmd_name = args_else[0] - - if cmd_name not in commands_dict: - guess = get_similar_commands(cmd_name) - - msg = ['unknown command "%s"' % cmd_name] - if guess: - msg.append('maybe you meant "%s"' % guess) - - raise CommandError(' - '.join(msg)) - - # all the args without the subcommand - cmd_args = args[:] - cmd_args.remove(cmd_name) - - return cmd_name, cmd_args - - -def check_isolated(args): - isolated = False - - if "--isolated" in args: - isolated = True - - return isolated - - def main(args=None): if args is None: args = sys.argv[1:] @@ -293,7 +61,7 @@ def main(args=None): autocomplete() try: - cmd_name, cmd_args = parseopts(args) + cmd_name, cmd_args = parse_command(args) except PipError as exc: sys.stderr.write("ERROR: %s" % exc) sys.stderr.write(os.linesep) @@ -306,5 +74,5 @@ def main(args=None): except locale.Error as e: # setlocale can apparently crash if locale are uninitialized logger.debug("Ignoring error %s when setting locale", e) - command = commands_dict[cmd_name](isolated=check_isolated(cmd_args)) + command = commands_dict[cmd_name](isolated=("--isolated" in cmd_args)) return command.main(cmd_args) diff --git a/pipenv/patched/notpip/_internal/build_env.py b/pipenv/patched/notpip/_internal/build_env.py index 1d351b5c..6d696fbd 100644 --- a/pipenv/patched/notpip/_internal/build_env.py +++ b/pipenv/patched/notpip/_internal/build_env.py @@ -7,6 +7,8 @@ import sys from distutils.sysconfig import get_python_lib from sysconfig import get_paths +from pipenv.patched.notpip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet + from pipenv.patched.notpip._internal.utils.misc import call_subprocess from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.utils.ui import open_spinner @@ -75,6 +77,20 @@ class BuildEnvironment(object): def cleanup(self): self._temp_dir.cleanup() + def missing_requirements(self, reqs): + """Return a list of the requirements from reqs that are not present + """ + missing = [] + with self: + ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep)) + for req in reqs: + try: + if ws.find(Requirement.parse(req)) is None: + missing.append(req) + except VersionConflict: + missing.append(req) + return missing + def install_requirements(self, finder, requirements, message): args = [ sys.executable, '-m', 'pip', 'install', '--ignore-installed', diff --git a/pipenv/patched/notpip/_internal/cache.py b/pipenv/patched/notpip/_internal/cache.py index a3a28fd8..d91b8170 100644 --- a/pipenv/patched/notpip/_internal/cache.py +++ b/pipenv/patched/notpip/_internal/cache.py @@ -8,9 +8,9 @@ import os from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal import index -from pipenv.patched.notpip._internal.compat import expanduser from pipenv.patched.notpip._internal.download import path_to_url +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.wheel import InvalidWheelFilename, Wheel @@ -22,7 +22,7 @@ class Cache(object): :param cache_dir: The root of the cache. - :param format_control: A pip.index.FormatControl object to limit + :param format_control: An object of FormatControl class to limit binaries being read from the cache. :param allowed_formats: which formats of files the cache should store. ('binary' and 'source' are the only allowed values) @@ -72,8 +72,8 @@ class Cache(object): return [] canonical_name = canonicalize_name(package_name) - formats = index.fmt_ctl_formats( - self.format_control, canonical_name + formats = self.format_control.get_allowed_formats( + canonical_name ) if not self.allowed_formats.intersection(formats): return [] @@ -101,7 +101,7 @@ class Cache(object): root = self.get_path_for_link(link) path = os.path.join(root, candidate) - return index.Link(path_to_url(path)) + return Link(path_to_url(path)) def cleanup(self): pass diff --git a/pipenv/patched/notpip/_internal/cli/__init__.py b/pipenv/patched/notpip/_internal/cli/__init__.py new file mode 100644 index 00000000..e589bb91 --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/__init__.py @@ -0,0 +1,4 @@ +"""Subpackage containing all of pip's command line interface related code +""" + +# This file intentionally does not import submodules diff --git a/pipenv/patched/notpip/_internal/cli/autocompletion.py b/pipenv/patched/notpip/_internal/cli/autocompletion.py new file mode 100644 index 00000000..15b560a1 --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/autocompletion.py @@ -0,0 +1,152 @@ +"""Logic that powers autocompletion installed by ``pip completion``. +""" + +import optparse +import os +import sys + +from pipenv.patched.notpip._internal.cli.main_parser import create_main_parser +from pipenv.patched.notpip._internal.commands import commands_dict, get_summaries +from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions + + +def autocomplete(): + """Entry Point for completion of main and subcommand options. + """ + # Don't complete if user hasn't sourced bash_completion file. + if 'PIP_AUTO_COMPLETE' not in os.environ: + return + cwords = os.environ['COMP_WORDS'].split()[1:] + cword = int(os.environ['COMP_CWORD']) + try: + current = cwords[cword - 1] + except IndexError: + current = '' + + subcommands = [cmd for cmd, summary in get_summaries()] + options = [] + # subcommand + try: + subcommand_name = [w for w in cwords if w in subcommands][0] + except IndexError: + subcommand_name = None + + parser = create_main_parser() + # subcommand options + if subcommand_name: + # special case: 'help' subcommand has no options + if subcommand_name == 'help': + sys.exit(1) + # special case: list locally installed dists for show and uninstall + should_list_installed = ( + subcommand_name in ['show', 'uninstall'] and + not current.startswith('-') + ) + if should_list_installed: + installed = [] + lc = current.lower() + for dist in get_installed_distributions(local_only=True): + if dist.key.startswith(lc) and dist.key not in cwords[1:]: + installed.append(dist.key) + # if there are no dists installed, fall back to option completion + if installed: + for dist in installed: + print(dist) + sys.exit(1) + + subcommand = commands_dict[subcommand_name]() + + for opt in subcommand.parser.option_list_all: + if opt.help != optparse.SUPPRESS_HELP: + for opt_str in opt._long_opts + opt._short_opts: + options.append((opt_str, opt.nargs)) + + # filter out previously specified options from available options + prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] + options = [(x, v) for (x, v) in options if x not in prev_opts] + # filter options by current input + options = [(k, v) for k, v in options if k.startswith(current)] + # get completion type given cwords and available subcommand options + completion_type = get_path_completion_type( + cwords, cword, subcommand.parser.option_list_all, + ) + # get completion files and directories if ``completion_type`` is + # ````, ```` or ```` + if completion_type: + options = auto_complete_paths(current, completion_type) + options = ((opt, 0) for opt in options) + for option in options: + opt_label = option[0] + # append '=' to options which require args + if option[1] and option[0][:2] == "--": + opt_label += '=' + print(opt_label) + else: + # show main parser options only when necessary + + opts = [i.option_list for i in parser.option_groups] + opts.append(parser.option_list) + opts = (o for it in opts for o in it) + if current.startswith('-'): + for opt in opts: + if opt.help != optparse.SUPPRESS_HELP: + subcommands += opt._long_opts + opt._short_opts + else: + # get completion type given cwords and all available options + completion_type = get_path_completion_type(cwords, cword, opts) + if completion_type: + subcommands = auto_complete_paths(current, completion_type) + + print(' '.join([x for x in subcommands if x.startswith(current)])) + sys.exit(1) + + +def get_path_completion_type(cwords, cword, opts): + """Get the type of path completion (``file``, ``dir``, ``path`` or None) + + :param cwords: same as the environmental variable ``COMP_WORDS`` + :param cword: same as the environmental variable ``COMP_CWORD`` + :param opts: The available options to check + :return: path completion type (``file``, ``dir``, ``path`` or None) + """ + if cword < 2 or not cwords[cword - 2].startswith('-'): + return + for opt in opts: + if opt.help == optparse.SUPPRESS_HELP: + continue + for o in str(opt).split('/'): + if cwords[cword - 2].split('=')[0] == o: + if not opt.metavar or any( + x in ('path', 'file', 'dir') + for x in opt.metavar.split('/')): + return opt.metavar + + +def auto_complete_paths(current, completion_type): + """If ``completion_type`` is ``file`` or ``path``, list all regular files + and directories starting with ``current``; otherwise only list directories + starting with ``current``. + + :param current: The word to be completed + :param completion_type: path completion type(`file`, `path` or `dir`)i + :return: A generator of regular files and/or directories + """ + directory, filename = os.path.split(current) + current_path = os.path.abspath(directory) + # Don't complete paths if they can't be accessed + if not os.access(current_path, os.R_OK): + return + filename = os.path.normcase(filename) + # list all files that start with ``filename`` + file_list = (x for x in os.listdir(current_path) + if os.path.normcase(x).startswith(filename)) + for f in file_list: + opt = os.path.join(current_path, f) + comp_file = os.path.normcase(os.path.join(directory, f)) + # complete regular files when there is not ```` after option + # complete directories when there is ````, ```` or + # ````after option + if completion_type != 'dir' and os.path.isfile(opt): + yield comp_file + elif os.path.isdir(opt): + yield os.path.join(comp_file, '') diff --git a/pipenv/patched/notpip/_internal/basecommand.py b/pipenv/patched/notpip/_internal/cli/base_command.py similarity index 92% rename from pipenv/patched/notpip/_internal/basecommand.py rename to pipenv/patched/notpip/_internal/cli/base_command.py index 60199d55..229831f2 100644 --- a/pipenv/patched/notpip/_internal/basecommand.py +++ b/pipenv/patched/notpip/_internal/cli/base_command.py @@ -7,10 +7,14 @@ import optparse import os import sys -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.baseparser import ( +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.parser import ( ConfigOptionParser, UpdatingDefaultsHelpFormatter, ) +from pipenv.patched.notpip._internal.cli.status_codes import ( + ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, + VIRTUALENV_NOT_FOUND, +) from pipenv.patched.notpip._internal.download import PipSession from pipenv.patched.notpip._internal.exceptions import ( BadCommand, CommandError, InstallationError, PreviousBuildDirError, @@ -18,12 +22,10 @@ from pipenv.patched.notpip._internal.exceptions import ( ) from pipenv.patched.notpip._internal.index import PackageFinder from pipenv.patched.notpip._internal.locations import running_under_virtualenv -from pipenv.patched.notpip._internal.req.req_file import parse_requirements -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement -from pipenv.patched.notpip._internal.status_codes import ( - ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, - VIRTUALENV_NOT_FOUND, +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, ) +from pipenv.patched.notpip._internal.req.req_file import parse_requirements from pipenv.patched.notpip._internal.utils.logging import setup_logging from pipenv.patched.notpip._internal.utils.misc import get_prog, normalize_path from pipenv.patched.notpip._internal.utils.outdated import pip_version_check @@ -168,12 +170,14 @@ class Command(object): return UNKNOWN_ERROR finally: - # Check if we're using the latest version of pip available - skip_version_check = ( - options.disable_pip_version_check or - getattr(options, "no_index", False) + allow_version_check = ( + # Does this command have the index_group options? + hasattr(options, "no_index") and + # Is this command allowed to perform this check? + not (options.disable_pip_version_check or options.no_index) ) - if not skip_version_check: + # Check if we're using the latest version of pip available + if allow_version_check: session = self._build_session( options, retries=0, @@ -208,7 +212,7 @@ class RequirementCommand(Command): requirement_set.add_requirement(req_to_add) for req in args: - req_to_add = InstallRequirement.from_line( + req_to_add = install_req_from_line( req, None, isolated=options.isolated_mode, wheel_cache=wheel_cache ) @@ -216,7 +220,7 @@ class RequirementCommand(Command): requirement_set.add_requirement(req_to_add) for req in options.editables: - req_to_add = InstallRequirement.from_editable( + req_to_add = install_req_from_editable( req, isolated=options.isolated_mode, wheel_cache=wheel_cache diff --git a/pipenv/patched/notpip/_internal/cmdoptions.py b/pipenv/patched/notpip/_internal/cli/cmdoptions.py similarity index 81% rename from pipenv/patched/notpip/_internal/cmdoptions.py rename to pipenv/patched/notpip/_internal/cli/cmdoptions.py index c25e769f..a075a67e 100644 --- a/pipenv/patched/notpip/_internal/cmdoptions.py +++ b/pipenv/patched/notpip/_internal/cli/cmdoptions.py @@ -13,10 +13,9 @@ import warnings from functools import partial from optparse import SUPPRESS_HELP, Option, OptionGroup -from pipenv.patched.notpip._internal.index import ( - FormatControl, fmt_ctl_handle_mutual_exclude, fmt_ctl_no_binary, -) +from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.locations import USER_CACHE_DIR, src_prefix +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.models.index import PyPI from pipenv.patched.notpip._internal.utils.hashes import STRONG_HASHES from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -53,13 +52,52 @@ def check_install_build_global(options, check_options=None): names = ["build_options", "global_options", "install_options"] if any(map(getname, names)): control = options.format_control - fmt_ctl_no_binary(control) + control.disallow_binaries() warnings.warn( 'Disabling all use of wheels due to the use of --build-options ' '/ --global-options / --install-options.', stacklevel=2, ) +def check_dist_restriction(options, check_target=False): + """Function for determining if custom platform options are allowed. + + :param options: The OptionParser options. + :param check_target: Whether or not to check if --target is being used. + """ + dist_restriction_set = any([ + options.python_version, + options.platform, + options.abi, + options.implementation, + ]) + + binary_only = FormatControl(set(), {':all:'}) + sdist_dependencies_allowed = ( + options.format_control != binary_only and + not options.ignore_dependencies + ) + + # Installations or downloads using dist restrictions must not combine + # source distributions and dist-specific wheels, as they are not + # gauranteed to be locally compatible. + if dist_restriction_set and sdist_dependencies_allowed: + raise CommandError( + "When restricting platform and interpreter constraints using " + "--python-version, --platform, --abi, or --implementation, " + "either --no-deps must be set, or --only-binary=:all: must be " + "set and --no-binary must not be set (or must be set to " + ":none:)." + ) + + if check_target: + if dist_restriction_set and not options.target_dir: + raise CommandError( + "Can not use any platform or abi specific options unless " + "installing via '--target'" + ) + + ########### # options # ########### @@ -365,24 +403,25 @@ def _get_format_control(values, option): def _handle_no_binary(option, opt_str, value, parser): - existing = getattr(parser.values, option.dest) - fmt_ctl_handle_mutual_exclude( + existing = _get_format_control(parser.values, option) + FormatControl.handle_mutual_excludes( value, existing.no_binary, existing.only_binary, ) def _handle_only_binary(option, opt_str, value, parser): - existing = getattr(parser.values, option.dest) - fmt_ctl_handle_mutual_exclude( + existing = _get_format_control(parser.values, option) + FormatControl.handle_mutual_excludes( value, existing.only_binary, existing.no_binary, ) def no_binary(): + format_control = FormatControl(set(), set()) return Option( "--no-binary", dest="format_control", action="callback", callback=_handle_no_binary, type="str", - default=FormatControl(set(), set()), + default=format_control, help="Do not use binary packages. Can be supplied multiple times, and " "each time adds to the existing value. Accepts either :all: to " "disable all binary packages, :none: to empty the set, or one or " @@ -393,10 +432,11 @@ def no_binary(): def only_binary(): + format_control = FormatControl(set(), set()) return Option( "--only-binary", dest="format_control", action="callback", callback=_handle_only_binary, type="str", - default=FormatControl(set(), set()), + default=format_control, help="Do not use source packages. Can be supplied multiple times, and " "each time adds to the existing value. Accepts either :all: to " "disable all source packages, :none: to empty the set, or one or " @@ -406,6 +446,61 @@ def only_binary(): ) +platform = partial( + Option, + '--platform', + dest='platform', + metavar='platform', + default=None, + help=("Only use wheels compatible with . " + "Defaults to the platform of the running system."), +) + + +python_version = partial( + Option, + '--python-version', + dest='python_version', + metavar='python_version', + default=None, + help=("Only use wheels compatible with Python " + "interpreter version . If not specified, then the " + "current system interpreter minor version is used. A major " + "version (e.g. '2') can be specified to match all " + "minor revs of that major version. A minor version " + "(e.g. '34') can also be specified."), +) + + +implementation = partial( + Option, + '--implementation', + dest='implementation', + metavar='implementation', + default=None, + help=("Only use wheels compatible with Python " + "implementation , e.g. 'pp', 'jy', 'cp', " + " or 'ip'. If not specified, then the current " + "interpreter implementation is used. Use 'py' to force " + "implementation-agnostic wheels."), +) + + +abi = partial( + Option, + '--abi', + dest='abi', + metavar='abi', + default=None, + help=("Only use wheels compatible with Python " + "abi , e.g. 'pypy_41'. If not specified, then the " + "current interpreter abi tag is used. Generally " + "you will need to specify --implementation, " + "--platform, and --python-version when using " + "this option."), +) + + def prefer_binary(): return Option( "--prefer-binary", @@ -501,7 +596,7 @@ no_clean = partial( '--no-clean', action='store_true', default=False, - help="Don't clean up build directories)." + help="Don't clean up build directories." ) # type: Any pre = partial( diff --git a/pipenv/patched/notpip/_internal/cli/main_parser.py b/pipenv/patched/notpip/_internal/cli/main_parser.py new file mode 100644 index 00000000..abe2f69e --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/main_parser.py @@ -0,0 +1,96 @@ +"""A single place for constructing and exposing the main parser +""" + +import os +import sys + +from pipenv.patched.notpip import __version__ +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.parser import ( + ConfigOptionParser, UpdatingDefaultsHelpFormatter, +) +from pipenv.patched.notpip._internal.commands import ( + commands_dict, get_similar_commands, get_summaries, +) +from pipenv.patched.notpip._internal.exceptions import CommandError +from pipenv.patched.notpip._internal.utils.misc import get_prog + +__all__ = ["create_main_parser", "parse_command"] + + +def create_main_parser(): + """Creates and returns the main parser for pip's CLI + """ + + parser_kw = { + 'usage': '\n%prog [options]', + 'add_help_option': False, + 'formatter': UpdatingDefaultsHelpFormatter(), + 'name': 'global', + 'prog': get_prog(), + } + + parser = ConfigOptionParser(**parser_kw) + parser.disable_interspersed_args() + + pip_pkg_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), "..", "..", + )) + parser.version = 'pip %s from %s (python %s)' % ( + __version__, pip_pkg_dir, sys.version[:3], + ) + + # add the general options + gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) + parser.add_option_group(gen_opts) + + parser.main = True # so the help formatter knows + + # create command listing for description + command_summaries = get_summaries() + description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries] + parser.description = '\n'.join(description) + + return parser + + +def parse_command(args): + parser = create_main_parser() + + # Note: parser calls disable_interspersed_args(), so the result of this + # call is to split the initial args into the general options before the + # subcommand and everything else. + # For example: + # args: ['--timeout=5', 'install', '--user', 'INITools'] + # general_options: ['--timeout==5'] + # args_else: ['install', '--user', 'INITools'] + general_options, args_else = parser.parse_args(args) + + # --version + if general_options.version: + sys.stdout.write(parser.version) + sys.stdout.write(os.linesep) + sys.exit() + + # pip || pip help -> print_help() + if not args_else or (args_else[0] == 'help' and len(args_else) == 1): + parser.print_help() + sys.exit() + + # the subcommand name + cmd_name = args_else[0] + + if cmd_name not in commands_dict: + guess = get_similar_commands(cmd_name) + + msg = ['unknown command "%s"' % cmd_name] + if guess: + msg.append('maybe you meant "%s"' % guess) + + raise CommandError(' - '.join(msg)) + + # all the args without the subcommand + cmd_args = args[:] + cmd_args.remove(cmd_name) + + return cmd_name, cmd_args diff --git a/pipenv/patched/notpip/_internal/baseparser.py b/pipenv/patched/notpip/_internal/cli/parser.py similarity index 88% rename from pipenv/patched/notpip/_internal/baseparser.py rename to pipenv/patched/notpip/_internal/cli/parser.py index f82093bf..2d2c8f4d 100644 --- a/pipenv/patched/notpip/_internal/baseparser.py +++ b/pipenv/patched/notpip/_internal/cli/parser.py @@ -9,8 +9,9 @@ from distutils.util import strtobool from pipenv.patched.notpip._vendor.six import string_types -from pipenv.patched.notpip._internal.compat import get_terminal_size +from pipenv.patched.notpip._internal.cli.status_codes import UNKNOWN_ERROR from pipenv.patched.notpip._internal.configuration import Configuration, ConfigurationError +from pipenv.patched.notpip._internal.utils.compat import get_terminal_size logger = logging.getLogger(__name__) @@ -192,7 +193,14 @@ class ConfigOptionParser(CustomOptionParser): continue if option.action in ('store_true', 'store_false', 'count'): - val = strtobool(val) + try: + val = strtobool(val) + except ValueError: + error_msg = invalid_config_error_message( + option.action, key, val + ) + self.error(error_msg) + elif option.action == 'append': val = val.split() val = [self.check_default(option, key, v) for v in val] @@ -225,7 +233,7 @@ class ConfigOptionParser(CustomOptionParser): try: self.config.load() except ConfigurationError as err: - self.exit(2, err.args[0]) + self.exit(UNKNOWN_ERROR, str(err)) defaults = self._update_defaults(self.defaults.copy()) # ours for option in self._get_all_options(): @@ -237,4 +245,17 @@ class ConfigOptionParser(CustomOptionParser): def error(self, msg): self.print_usage(sys.stderr) - self.exit(2, "%s\n" % msg) + self.exit(UNKNOWN_ERROR, "%s\n" % msg) + + +def invalid_config_error_message(action, key, val): + """Returns a better error message when invalid configuration option + is provided.""" + if action in ('store_true', 'store_false'): + return ("{0} is not a valid value for {1} option, " + "please specify a boolean value like yes/no, " + "true/false or 1/0 instead.").format(val, key) + + return ("{0} is not a valid value for {1} option, " + "please specify a numerical value like 1/0 " + "instead.").format(val, key) diff --git a/pipenv/patched/notpip/_internal/status_codes.py b/pipenv/patched/notpip/_internal/cli/status_codes.py similarity index 100% rename from pipenv/patched/notpip/_internal/status_codes.py rename to pipenv/patched/notpip/_internal/cli/status_codes.py diff --git a/pipenv/patched/notpip/_internal/commands/__init__.py b/pipenv/patched/notpip/_internal/commands/__init__.py index 140c4609..a403c6f9 100644 --- a/pipenv/patched/notpip/_internal/commands/__init__.py +++ b/pipenv/patched/notpip/_internal/commands/__init__.py @@ -21,7 +21,7 @@ from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import List, Type # noqa: F401 - from pipenv.patched.notpip._internal.basecommand import Command # noqa: F401 + from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 commands_order = [ InstallCommand, diff --git a/pipenv/patched/notpip/_internal/commands/check.py b/pipenv/patched/notpip/_internal/commands/check.py index cd5ffb5f..adf4f5e7 100644 --- a/pipenv/patched/notpip/_internal/commands/check.py +++ b/pipenv/patched/notpip/_internal/commands/check.py @@ -1,6 +1,6 @@ import logging -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.operations.check import ( check_package_set, create_package_set_from_installed, ) diff --git a/pipenv/patched/notpip/_internal/commands/completion.py b/pipenv/patched/notpip/_internal/commands/completion.py index f4c31c1b..cb8a11a7 100644 --- a/pipenv/patched/notpip/_internal/commands/completion.py +++ b/pipenv/patched/notpip/_internal/commands/completion.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import sys import textwrap -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.utils.misc import get_prog BASE_COMPLETION = """ diff --git a/pipenv/patched/notpip/_internal/commands/configuration.py b/pipenv/patched/notpip/_internal/commands/configuration.py index 090109c5..6c1dbdfd 100644 --- a/pipenv/patched/notpip/_internal/commands/configuration.py +++ b/pipenv/patched/notpip/_internal/commands/configuration.py @@ -2,11 +2,11 @@ import logging import os import subprocess -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR, SUCCESS from pipenv.patched.notpip._internal.configuration import Configuration, kinds from pipenv.patched.notpip._internal.exceptions import PipError from pipenv.patched.notpip._internal.locations import venv_config_file -from pipenv.patched.notpip._internal.status_codes import ERROR, SUCCESS from pipenv.patched.notpip._internal.utils.misc import get_prog logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/download.py b/pipenv/patched/notpip/_internal/commands/download.py index 63d91b04..e5d87121 100644 --- a/pipenv/patched/notpip/_internal/commands/download.py +++ b/pipenv/patched/notpip/_internal/commands/download.py @@ -3,10 +3,8 @@ from __future__ import absolute_import import logging import os -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand -from pipenv.patched.notpip._internal.exceptions import CommandError -from pipenv.patched.notpip._internal.index import FormatControl +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer from pipenv.patched.notpip._internal.req import RequirementSet from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker @@ -69,52 +67,10 @@ class DownloadCommand(RequirementCommand): help=("Download packages into ."), ) - cmd_opts.add_option( - '--platform', - dest='platform', - metavar='platform', - default=None, - help=("Only download wheels compatible with . " - "Defaults to the platform of the running system."), - ) - - cmd_opts.add_option( - '--python-version', - dest='python_version', - metavar='python_version', - default=None, - help=("Only download wheels compatible with Python " - "interpreter version . If not specified, then the " - "current system interpreter minor version is used. A major " - "version (e.g. '2') can be specified to match all " - "minor revs of that major version. A minor version " - "(e.g. '34') can also be specified."), - ) - - cmd_opts.add_option( - '--implementation', - dest='implementation', - metavar='implementation', - default=None, - help=("Only download wheels compatible with Python " - "implementation , e.g. 'pp', 'jy', 'cp', " - " or 'ip'. If not specified, then the current " - "interpreter implementation is used. Use 'py' to force " - "implementation-agnostic wheels."), - ) - - cmd_opts.add_option( - '--abi', - dest='abi', - metavar='abi', - default=None, - help=("Only download wheels compatible with Python " - "abi , e.g. 'pypy_41'. If not specified, then the " - "current interpreter abi tag is used. Generally " - "you will need to specify --implementation, " - "--platform, and --python-version when using " - "this option."), - ) + cmd_opts.add_option(cmdoptions.platform()) + cmd_opts.add_option(cmdoptions.python_version()) + cmd_opts.add_option(cmdoptions.implementation()) + cmd_opts.add_option(cmdoptions.abi()) index_opts = cmdoptions.make_option_group( cmdoptions.index_group, @@ -135,25 +91,7 @@ class DownloadCommand(RequirementCommand): else: python_versions = None - dist_restriction_set = any([ - options.python_version, - options.platform, - options.abi, - options.implementation, - ]) - binary_only = FormatControl(set(), {':all:'}) - no_sdist_dependencies = ( - options.format_control != binary_only and - not options.ignore_dependencies - ) - if dist_restriction_set and no_sdist_dependencies: - raise CommandError( - "When restricting platform and interpreter constraints using " - "--python-version, --platform, --abi, or --implementation, " - "either --no-deps must be set, or --only-binary=:all: must be " - "set and --no-binary must not be set (or must be set to " - ":none:)." - ) + cmdoptions.check_dist_restriction(options) options.src_dir = os.path.abspath(options.src_dir) options.download_dir = normalize_path(options.download_dir) diff --git a/pipenv/patched/notpip/_internal/commands/freeze.py b/pipenv/patched/notpip/_internal/commands/freeze.py index 6959b6ab..343227ba 100644 --- a/pipenv/patched/notpip/_internal/commands/freeze.py +++ b/pipenv/patched/notpip/_internal/commands/freeze.py @@ -2,11 +2,11 @@ from __future__ import absolute_import import sys -from pipenv.patched.notpip._internal import index -from pipenv.patched.notpip._internal.basecommand import Command from pipenv.patched.notpip._internal.cache import WheelCache -from pipenv.patched.notpip._internal.compat import stdlib_pkgs +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.operations.freeze import freeze +from pipenv.patched.notpip._internal.utils.compat import stdlib_pkgs DEV_PKGS = {'pip', 'setuptools', 'distribute', 'wheel'} @@ -71,7 +71,7 @@ class FreezeCommand(Command): self.parser.insert_option_group(0, self.cmd_opts) def run(self, options, args): - format_control = index.FormatControl(set(), set()) + format_control = FormatControl(set(), set()) wheel_cache = WheelCache(options.cache_dir, format_control) skip = set(stdlib_pkgs) if not options.freeze_all: diff --git a/pipenv/patched/notpip/_internal/commands/hash.py b/pipenv/patched/notpip/_internal/commands/hash.py index a86574da..183f11ae 100644 --- a/pipenv/patched/notpip/_internal/commands/hash.py +++ b/pipenv/patched/notpip/_internal/commands/hash.py @@ -4,8 +4,8 @@ import hashlib import logging import sys -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.status_codes import ERROR +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR from pipenv.patched.notpip._internal.utils.hashes import FAVORITE_HASH, STRONG_HASHES from pipenv.patched.notpip._internal.utils.misc import read_chunks diff --git a/pipenv/patched/notpip/_internal/commands/help.py b/pipenv/patched/notpip/_internal/commands/help.py index aaf0f87e..f2c61965 100644 --- a/pipenv/patched/notpip/_internal/commands/help.py +++ b/pipenv/patched/notpip/_internal/commands/help.py @@ -1,6 +1,7 @@ from __future__ import absolute_import -from pipenv.patched.notpip._internal.basecommand import SUCCESS, Command +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import SUCCESS from pipenv.patched.notpip._internal.exceptions import CommandError diff --git a/pipenv/patched/notpip/_internal/commands/install.py b/pipenv/patched/notpip/_internal/commands/install.py index ebdf07d7..ddcb4759 100644 --- a/pipenv/patched/notpip/_internal/commands/install.py +++ b/pipenv/patched/notpip/_internal/commands/install.py @@ -9,9 +9,10 @@ from optparse import SUPPRESS_HELP from pipenv.patched.notpip._vendor import pkg_resources -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand from pipenv.patched.notpip._internal.cache import WheelCache +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand +from pipenv.patched.notpip._internal.cli.status_codes import ERROR from pipenv.patched.notpip._internal.exceptions import ( CommandError, InstallationError, PreviousBuildDirError, ) @@ -21,7 +22,6 @@ from pipenv.patched.notpip._internal.operations.prepare import RequirementPrepar from pipenv.patched.notpip._internal.req import RequirementSet, install_given_reqs from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker from pipenv.patched.notpip._internal.resolve import Resolver -from pipenv.patched.notpip._internal.status_codes import ERROR from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner from pipenv.patched.notpip._internal.utils.misc import ( ensure_dir, get_installed_version, @@ -83,6 +83,11 @@ class InstallCommand(RequirementCommand): '. Use --upgrade to replace existing packages in ' 'with new versions.' ) + cmd_opts.add_option(cmdoptions.platform()) + cmd_opts.add_option(cmdoptions.python_version()) + cmd_opts.add_option(cmdoptions.implementation()) + cmd_opts.add_option(cmdoptions.abi()) + cmd_opts.add_option( '--user', dest='use_user_site', @@ -204,7 +209,6 @@ class InstallCommand(RequirementCommand): def run(self, options, args): cmdoptions.check_install_build_global(options) - upgrade_strategy = "to-satisfy-only" if options.upgrade: upgrade_strategy = options.upgrade_strategy @@ -212,6 +216,13 @@ class InstallCommand(RequirementCommand): if options.build_dir: options.build_dir = os.path.abspath(options.build_dir) + cmdoptions.check_dist_restriction(options, check_target=True) + + if options.python_version: + python_versions = [options.python_version] + else: + python_versions = None + options.src_dir = os.path.abspath(options.src_dir) install_options = options.install_options or [] if options.use_user_site: @@ -246,7 +257,14 @@ class InstallCommand(RequirementCommand): global_options = options.global_options or [] with self._build_session(options) as session: - finder = self._build_package_finder(options, session) + finder = self._build_package_finder( + options=options, + session=session, + platform=options.platform, + python_versions=python_versions, + abi=options.abi, + implementation=options.implementation, + ) build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) @@ -266,6 +284,7 @@ class InstallCommand(RequirementCommand): ) as directory: requirement_set = RequirementSet( require_hashes=options.require_hashes, + check_supported_wheels=not options.target_dir, ) try: diff --git a/pipenv/patched/notpip/_internal/commands/list.py b/pipenv/patched/notpip/_internal/commands/list.py index 99aee99f..577c0b5f 100644 --- a/pipenv/patched/notpip/_internal/commands/list.py +++ b/pipenv/patched/notpip/_internal/commands/list.py @@ -6,8 +6,8 @@ import logging from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import zip_longest -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.cmdoptions import index_group, make_option_group +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.index import PackageFinder from pipenv.patched.notpip._internal.utils.misc import ( @@ -102,7 +102,9 @@ class ListCommand(Command): help='Include editable package from output.', default=True, ) - index_opts = make_option_group(index_group, self.parser) + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, self.parser + ) self.parser.insert_option_group(0, index_opts) self.parser.insert_option_group(0, cmd_opts) diff --git a/pipenv/patched/notpip/_internal/commands/search.py b/pipenv/patched/notpip/_internal/commands/search.py index ac111c14..986208f9 100644 --- a/pipenv/patched/notpip/_internal/commands/search.py +++ b/pipenv/patched/notpip/_internal/commands/search.py @@ -11,12 +11,12 @@ from pipenv.patched.notpip._vendor.packaging.version import parse as parse_versi # why we ignore the type on this import from pipenv.patched.notpip._vendor.six.moves import xmlrpc_client # type: ignore -from pipenv.patched.notpip._internal.basecommand import SUCCESS, Command -from pipenv.patched.notpip._internal.compat import get_terminal_size +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import NO_MATCHES_FOUND, SUCCESS from pipenv.patched.notpip._internal.download import PipXmlrpcTransport from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.models.index import PyPI -from pipenv.patched.notpip._internal.status_codes import NO_MATCHES_FOUND +from pipenv.patched.notpip._internal.utils.compat import get_terminal_size from pipenv.patched.notpip._internal.utils.logging import indent_log logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/show.py b/pipenv/patched/notpip/_internal/commands/show.py index 8de6b6b8..3fd24482 100644 --- a/pipenv/patched/notpip/_internal/commands/show.py +++ b/pipenv/patched/notpip/_internal/commands/show.py @@ -7,8 +7,8 @@ from email.parser import FeedParser # type: ignore from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.status_codes import ERROR, SUCCESS +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR, SUCCESS logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/uninstall.py b/pipenv/patched/notpip/_internal/commands/uninstall.py index 45a0eba5..cf6a511c 100644 --- a/pipenv/patched/notpip/_internal/commands/uninstall.py +++ b/pipenv/patched/notpip/_internal/commands/uninstall.py @@ -2,9 +2,10 @@ from __future__ import absolute_import from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.exceptions import InstallationError -from pipenv.patched.notpip._internal.req import InstallRequirement, parse_requirements +from pipenv.patched.notpip._internal.req import parse_requirements +from pipenv.patched.notpip._internal.req.constructors import install_req_from_line from pipenv.patched.notpip._internal.utils.misc import protect_pip_from_modification_on_windows @@ -47,7 +48,7 @@ class UninstallCommand(Command): with self._build_session(options) as session: reqs_to_uninstall = {} for name in args: - req = InstallRequirement.from_line( + req = install_req_from_line( name, isolated=options.isolated_mode, ) if req.name: diff --git a/pipenv/patched/notpip/_internal/commands/wheel.py b/pipenv/patched/notpip/_internal/commands/wheel.py index c04d58ed..08d695ab 100644 --- a/pipenv/patched/notpip/_internal/commands/wheel.py +++ b/pipenv/patched/notpip/_internal/commands/wheel.py @@ -4,9 +4,9 @@ from __future__ import absolute_import import logging import os -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand from pipenv.patched.notpip._internal.cache import WheelCache +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand from pipenv.patched.notpip._internal.exceptions import CommandError, PreviousBuildDirError from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer from pipenv.patched.notpip._internal.req import RequirementSet diff --git a/pipenv/patched/notpip/_internal/configuration.py b/pipenv/patched/notpip/_internal/configuration.py index 3df185f7..3c02c955 100644 --- a/pipenv/patched/notpip/_internal/configuration.py +++ b/pipenv/patched/notpip/_internal/configuration.py @@ -18,7 +18,9 @@ import os from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import configparser -from pipenv.patched.notpip._internal.exceptions import ConfigurationError +from pipenv.patched.notpip._internal.exceptions import ( + ConfigurationError, ConfigurationFileCouldNotBeLoaded, +) from pipenv.patched.notpip._internal.locations import ( legacy_config_file, new_config_file, running_under_virtualenv, site_config_files, venv_config_file, @@ -289,11 +291,16 @@ class Configuration(object): try: parser.read(fname) except UnicodeDecodeError: - raise ConfigurationError(( - "ERROR: " - "Configuration file contains invalid %s characters.\n" - "Please fix your configuration, located at %s\n" - ) % (locale.getpreferredencoding(False), fname)) + # See https://github.com/pypa/pip/issues/4963 + raise ConfigurationFileCouldNotBeLoaded( + reason="contains invalid {} characters".format( + locale.getpreferredencoding(False) + ), + fname=fname, + ) + except configparser.Error as error: + # See https://github.com/pypa/pip/issues/4893 + raise ConfigurationFileCouldNotBeLoaded(error=error) return parser def _load_environment_vars(self): diff --git a/pipenv/patched/notpip/_internal/download.py b/pipenv/patched/notpip/_internal/download.py index 06a45644..8f4c38f5 100644 --- a/pipenv/patched/notpip/_internal/download.py +++ b/pipenv/patched/notpip/_internal/download.py @@ -323,7 +323,7 @@ class InsecureHTTPAdapter(HTTPAdapter): conn.ca_certs = None -class PipSession(Session): +class PipSession(requests.Session): timeout = None @@ -753,7 +753,7 @@ def _copy_dist_from_dir(link_path, location): # build an sdist setup_py = 'setup.py' - sdist_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable)] + sdist_args = [sys.executable] sdist_args.append('-c') sdist_args.append(SETUPTOOLS_SHIM % setup_py) sdist_args.append('sdist') diff --git a/pipenv/patched/notpip/_internal/exceptions.py b/pipenv/patched/notpip/_internal/exceptions.py index 43595977..2eadcf28 100644 --- a/pipenv/patched/notpip/_internal/exceptions.py +++ b/pipenv/patched/notpip/_internal/exceptions.py @@ -247,3 +247,22 @@ class HashMismatch(HashError): class UnsupportedPythonVersion(InstallationError): """Unsupported python version according to Requires-Python package metadata.""" + + +class ConfigurationFileCouldNotBeLoaded(ConfigurationError): + """When there are errors while loading a configuration file + """ + + def __init__(self, reason="could not be loaded", fname=None, error=None): + super(ConfigurationFileCouldNotBeLoaded, self).__init__(error) + self.reason = reason + self.fname = fname + self.error = error + + def __str__(self): + if self.fname is not None: + message_part = " in {}.".format(self.fname) + else: + assert self.error is not None + message_part = ".\n{}\n".format(self.error.message) + return "Configuration file {}{}".format(self.reason, message_part) diff --git a/pipenv/patched/notpip/_internal/index.py b/pipenv/patched/notpip/_internal/index.py index 426880e9..b4b02373 100644 --- a/pipenv/patched/notpip/_internal/index.py +++ b/pipenv/patched/notpip/_internal/index.py @@ -20,24 +20,27 @@ from pipenv.patched.notpip._vendor.requests.exceptions import SSLError from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._internal.compat import ipaddress from pipenv.patched.notpip._internal.download import HAS_TLS, is_url, path_to_url, url_to_path from pipenv.patched.notpip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, InvalidWheelFilename, UnsupportedWheel, ) +from pipenv.patched.notpip._internal.models.candidate import InstallationCandidate +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.models.index import PyPI +from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.pep425tags import get_supported +from pipenv.patched.notpip._internal.utils.compat import ipaddress from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, cached_property, normalize_path, - remove_auth_from_url, splitext, + ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, normalize_path, + remove_auth_from_url, ) from pipenv.patched.notpip._internal.utils.packaging import check_requires_python from pipenv.patched.notpip._internal.wheel import Wheel, wheel_ext -__all__ = ['FormatControl', 'fmt_ctl_handle_mutual_exclude', 'PackageFinder'] +__all__ = ['FormatControl', 'PackageFinder'] SECURE_ORIGINS = [ @@ -56,46 +59,120 @@ SECURE_ORIGINS = [ logger = logging.getLogger(__name__) -class InstallationCandidate(object): +def _get_content_type(url, session): + """Get the Content-Type of the given url, using a HEAD request""" + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) + if scheme not in {'http', 'https'}: + # FIXME: some warning or something? + # assertion error? + return '' - def __init__(self, project, version, location, requires_python=None): - self.project = project - self.version = parse_version(version) - self.location = location - self._key = (self.project, self.version, self.location) - self.requires_python = requires_python + resp = session.head(url, allow_redirects=True) + resp.raise_for_status() - def __repr__(self): - return "".format( - self.project, self.version, self.location, + return resp.headers.get("Content-Type", "") + + +def _handle_get_page_fail(link, reason, url, meth=None): + if meth is None: + meth = logger.debug + meth("Could not fetch URL %s: %s - skipping", link, reason) + + +def _get_html_page(link, session=None): + if session is None: + raise TypeError( + "_get_html_page() missing 1 required keyword argument: 'session'" ) - def __hash__(self): - return hash(self._key) + url = link.url + url = url.split('#', 1)[0] - def __lt__(self, other): - return self._compare(other, lambda s, o: s < o) + # Check for VCS schemes that do not support lookup as web pages. + from pipenv.patched.notpip._internal.vcs import VcsSupport + for scheme in VcsSupport.schemes: + if url.lower().startswith(scheme) and url[len(scheme)] in '+:': + logger.debug('Cannot look at %s URL %s', scheme, link) + return None - def __le__(self, other): - return self._compare(other, lambda s, o: s <= o) + try: + filename = link.filename + for bad_ext in ARCHIVE_EXTENSIONS: + if filename.endswith(bad_ext): + content_type = _get_content_type(url, session=session) + if content_type.lower().startswith('text/html'): + break + else: + logger.debug( + 'Skipping page %s because of Content-Type: %s', + link, + content_type, + ) + return - def __eq__(self, other): - return self._compare(other, lambda s, o: s == o) + logger.debug('Getting page %s', url) - def __ge__(self, other): - return self._compare(other, lambda s, o: s >= o) + # Tack index.html onto file:// URLs that point to directories + (scheme, netloc, path, params, query, fragment) = \ + urllib_parse.urlparse(url) + if (scheme == 'file' and + os.path.isdir(urllib_request.url2pathname(path))): + # add trailing slash if not present so urljoin doesn't trim + # final segment + if not url.endswith('/'): + url += '/' + url = urllib_parse.urljoin(url, 'index.html') + logger.debug(' file: URL is directory, getting %s', url) - def __gt__(self, other): - return self._compare(other, lambda s, o: s > o) + resp = session.get( + url, + headers={ + "Accept": "text/html", + # We don't want to blindly returned cached data for + # /simple/, because authors generally expecting that + # twine upload && pip install will function, but if + # they've done a pip install in the last ~10 minutes + # it won't. Thus by setting this to zero we will not + # blindly use any cached data, however the benefit of + # using max-age=0 instead of no-cache, is that we will + # still support conditional requests, so we will still + # minimize traffic sent in cases where the page hasn't + # changed at all, we will just always incur the round + # trip for the conditional GET now instead of only + # once per 10 minutes. + # For more information, please see pypa/pip#5670. + "Cache-Control": "max-age=0", + }, + ) + resp.raise_for_status() - def __ne__(self, other): - return self._compare(other, lambda s, o: s != o) + # The check for archives above only works if the url ends with + # something that looks like an archive. However that is not a + # requirement of an url. Unless we issue a HEAD request on every + # url we cannot know ahead of time for sure if something is HTML + # or not. However we can check after we've downloaded it. + content_type = resp.headers.get('Content-Type', 'unknown') + if not content_type.lower().startswith("text/html"): + logger.debug( + 'Skipping page %s because of Content-Type: %s', + link, + content_type, + ) + return - def _compare(self, other, method): - if not isinstance(other, InstallationCandidate): - return NotImplemented - - return method(self._key, other._key) + inst = HTMLPage(resp.content, resp.url, resp.headers) + except requests.HTTPError as exc: + _handle_get_page_fail(link, exc, url) + except SSLError as exc: + reason = "There was a problem confirming the ssl certificate: " + reason += str(exc) + _handle_get_page_fail(link, reason, url, meth=logger.info) + except requests.ConnectionError as exc: + _handle_get_page_fail(link, "connection error: %s" % exc, url) + except requests.Timeout: + _handle_get_page_fail(link, "timed out", url) + else: + return inst class PackageFinder(object): @@ -210,15 +287,15 @@ class PackageFinder(object): return "\n".join(lines) def add_dependency_links(self, links): - # # FIXME: this shouldn't be global list this, it should only - # # apply to requirements of the package that specifies the - # # dependency_links value - # # FIXME: also, we should track comes_from (i.e., use Link) + # FIXME: this shouldn't be global list this, it should only + # apply to requirements of the package that specifies the + # dependency_links value + # FIXME: also, we should track comes_from (i.e., use Link) if self.process_dependency_links: deprecated( "Dependency Links processing has been deprecated and will be " "removed in a future release.", - replacement=None, + replacement="PEP 508 URL dependencies", gone_in="18.2", issue=4187, ) @@ -242,6 +319,7 @@ class PackageFinder(object): return extras + @staticmethod def _sort_locations(locations, expand_dir=False): """ @@ -467,7 +545,7 @@ class PackageFinder(object): logger.debug('* %s', location) canonical_name = canonicalize_name(project_name) - formats = fmt_ctl_formats(self.format_control, canonical_name) + formats = self.format_control.get_allowed_formats(canonical_name) search = Search(project_name, canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links @@ -483,7 +561,7 @@ class PackageFinder(object): continue with indent_log(): page_versions.extend( - self._package_versions(page.links, search) + self._package_versions(page.iter_links(), search) ) dependency_versions = self._package_versions( @@ -626,7 +704,9 @@ class PackageFinder(object): try: page = self._get_page(location) - except requests.HTTPError as e: + except requests.HTTPError: + continue + if page is None: continue yield page @@ -743,7 +823,7 @@ class PackageFinder(object): return InstallationCandidate(search.supplied, version, link, link.requires_python) def _get_page(self, link): - return HTMLPage.get_page(link, session=self.session) + return _get_html_page(link, session=self.session) def egg_info_matches( @@ -763,7 +843,7 @@ def egg_info_matches( return None if search_name is None: full_match = match.group(0) - return full_match[full_match.index('-'):] + return full_match.split('-', 1)[-1] name = match.group(0).lower() # To match the "safe" name that pkg_resources creates: name = name.replace('_', '-') @@ -775,377 +855,71 @@ def egg_info_matches( return None +def _determine_base_url(document, page_url): + """Determine the HTML document's base URL. + + This looks for a ```` tag in the HTML document. If present, its href + attribute denotes the base URL of anchor tags in the document. If there is + no such tag (or if it does not have a valid href attribute), the HTML + file's URL is used as the base URL. + + :param document: An HTML document representation. The current + implementation expects the result of ``html5lib.parse()``. + :param page_url: The URL of the HTML document. + """ + for base in document.findall(".//base"): + href = base.get("href") + if href is not None: + return href + return page_url + + +def _get_encoding_from_headers(headers): + """Determine if we have any encoding information in our headers. + """ + if headers and "Content-Type" in headers: + content_type, params = cgi.parse_header(headers["Content-Type"]) + if "charset" in params: + return params['charset'] + return None + + +_CLEAN_LINK_RE = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) + + +def _clean_link(url): + """Makes sure a link is fully encoded. That is, if a ' ' shows up in + the link, it will be rewritten to %20 (while not over-quoting + % or other characters).""" + return _CLEAN_LINK_RE.sub(lambda match: '%%%2x' % ord(match.group(0)), url) + + class HTMLPage(object): """Represents one page, along with its URL""" def __init__(self, content, url, headers=None): - # Determine if we have any encoding information in our headers - encoding = None - if headers and "Content-Type" in headers: - content_type, params = cgi.parse_header(headers["Content-Type"]) - - if "charset" in params: - encoding = params['charset'] - self.content = content - self.parsed = html5lib.parse( - self.content, - transport_encoding=encoding, - namespaceHTMLElements=False, - ) self.url = url self.headers = headers def __str__(self): return self.url - @classmethod - def get_page(cls, link, skip_archives=True, session=None): - if session is None: - raise TypeError( - "get_page() missing 1 required keyword argument: 'session'" - ) - - url = link.url - url = url.split('#', 1)[0] - - # Check for VCS schemes that do not support lookup as web pages. - from pipenv.patched.notpip._internal.vcs import VcsSupport - for scheme in VcsSupport.schemes: - if url.lower().startswith(scheme) and url[len(scheme)] in '+:': - logger.debug('Cannot look at %s URL %s', scheme, link) - return None - - try: - if skip_archives: - filename = link.filename - for bad_ext in ARCHIVE_EXTENSIONS: - if filename.endswith(bad_ext): - content_type = cls._get_content_type( - url, session=session, - ) - if content_type.lower().startswith('text/html'): - break - else: - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - logger.debug('Getting page %s', url) - - # Tack index.html onto file:// URLs that point to directories - (scheme, netloc, path, params, query, fragment) = \ - urllib_parse.urlparse(url) - if (scheme == 'file' and - os.path.isdir(urllib_request.url2pathname(path))): - # add trailing slash if not present so urljoin doesn't trim - # final segment - if not url.endswith('/'): - url += '/' - url = urllib_parse.urljoin(url, 'index.html') - logger.debug(' file: URL is directory, getting %s', url) - - resp = session.get( - url, - headers={ - "Accept": "text/html", - "Cache-Control": "max-age=600", - }, - ) - resp.raise_for_status() - - # The check for archives above only works if the url ends with - # something that looks like an archive. However that is not a - # requirement of an url. Unless we issue a HEAD request on every - # url we cannot know ahead of time for sure if something is HTML - # or not. However we can check after we've downloaded it. - content_type = resp.headers.get('Content-Type', 'unknown') - if not content_type.lower().startswith("text/html"): - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - inst = cls(resp.content, resp.url, resp.headers) - except requests.HTTPError as exc: - cls._handle_fail(link, exc, url) - except SSLError as exc: - reason = "There was a problem confirming the ssl certificate: " - reason += str(exc) - cls._handle_fail(link, reason, url, meth=logger.info) - except requests.ConnectionError as exc: - cls._handle_fail(link, "connection error: %s" % exc, url) - except requests.Timeout: - cls._handle_fail(link, "timed out", url) - else: - return inst - - @staticmethod - def _handle_fail(link, reason, url, meth=None): - if meth is None: - meth = logger.debug - - meth("Could not fetch URL %s: %s - skipping", link, reason) - - @staticmethod - def _get_content_type(url, session): - """Get the Content-Type of the given url, using a HEAD request""" - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) - if scheme not in {'http', 'https'}: - # FIXME: some warning or something? - # assertion error? - return '' - - resp = session.head(url, allow_redirects=True) - resp.raise_for_status() - - return resp.headers.get("Content-Type", "") - - @cached_property - def base_url(self): - bases = [ - x for x in self.parsed.findall(".//base") - if x.get("href") is not None - ] - if bases and bases[0].get("href"): - return bases[0].get("href") - else: - return self.url - - @property - def links(self): + def iter_links(self): """Yields all links in the page""" - for anchor in self.parsed.findall(".//a"): + document = html5lib.parse( + self.content, + transport_encoding=_get_encoding_from_headers(self.headers), + namespaceHTMLElements=False, + ) + base_url = _determine_base_url(document, self.url) + for anchor in document.findall(".//a"): if anchor.get("href"): href = anchor.get("href") - url = self.clean_link( - urllib_parse.urljoin(self.base_url, href) - ) + url = _clean_link(urllib_parse.urljoin(base_url, href)) pyrequire = anchor.get('data-requires-python') pyrequire = unescape(pyrequire) if pyrequire else None - yield Link(url, self, requires_python=pyrequire) - - _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) - - def clean_link(self, url): - """Makes sure a link is fully encoded. That is, if a ' ' shows up in - the link, it will be rewritten to %20 (while not over-quoting - % or other characters).""" - return self._clean_re.sub( - lambda match: '%%%2x' % ord(match.group(0)), url) - - -class Link(object): - - def __init__(self, url, comes_from=None, requires_python=None): - """ - Object representing a parsed link from https://pypi.org/simple/* - - url: - url of the resource pointed to (href of the link) - comes_from: - instance of HTMLPage where the link was found, or string. - requires_python: - String containing the `Requires-Python` metadata field, specified - in PEP 345. This may be specified by a data-requires-python - attribute in the HTML link tag, as described in PEP 503. - """ - - # url can be a UNC windows share - if url.startswith('\\\\'): - url = path_to_url(url) - - self.url = url - self.comes_from = comes_from - self.requires_python = requires_python if requires_python else None - - def __str__(self): - if self.requires_python: - rp = ' (requires-python:%s)' % self.requires_python - else: - rp = '' - if self.comes_from: - return '%s (from %s)%s' % (self.url, self.comes_from, rp) - else: - return str(self.url) - - def __repr__(self): - return '' % self - - def __eq__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url == other.url - - def __ne__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url != other.url - - def __lt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url < other.url - - def __le__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url <= other.url - - def __gt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url > other.url - - def __ge__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url >= other.url - - def __hash__(self): - return hash(self.url) - - @property - def filename(self): - _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) - name = posixpath.basename(path.rstrip('/')) or netloc - name = urllib_parse.unquote(name) - assert name, ('URL %r produced no filename' % self.url) - return name - - @property - def scheme(self): - return urllib_parse.urlsplit(self.url)[0] - - @property - def netloc(self): - return urllib_parse.urlsplit(self.url)[1] - - @property - def path(self): - return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) - - def splitext(self): - return splitext(posixpath.basename(self.path.rstrip('/'))) - - @property - def ext(self): - return self.splitext()[1] - - @property - def url_without_fragment(self): - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) - return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) - - _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') - - @property - def egg_fragment(self): - match = self._egg_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') - - @property - def subdirectory_fragment(self): - match = self._subdirectory_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _hash_re = re.compile( - r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' - ) - - @property - def hash(self): - match = self._hash_re.search(self.url) - if match: - return match.group(2) - return None - - @property - def hash_name(self): - match = self._hash_re.search(self.url) - if match: - return match.group(1) - return None - - @property - def show_url(self): - return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) - - @property - def is_wheel(self): - return self.ext == wheel_ext - - @property - def is_artifact(self): - """ - Determines if this points to an actual artifact (e.g. a tarball) or if - it points to an "abstract" thing like a path or a VCS location. - """ - from pipenv.patched.notpip._internal.vcs import vcs - - if self.scheme in vcs.all_schemes: - return False - - return True - - -FormatControl = namedtuple('FormatControl', 'no_binary only_binary') -"""This object has two fields, no_binary and only_binary. - -If a field is falsy, it isn't set. If it is {':all:'}, it should match all -packages except those listed in the other field. Only one field can be set -to {':all:'} at a time. The rest of the time exact package name matches -are listed, with any given package only showing up in one field at a time. -""" - - -def fmt_ctl_handle_mutual_exclude(value, target, other): - new = value.split(',') - while ':all:' in new: - other.clear() - target.clear() - target.add(':all:') - del new[:new.index(':all:') + 1] - if ':none:' not in new: - # Without a none, we want to discard everything as :all: covers it - return - for name in new: - if name == ':none:': - target.clear() - continue - name = canonicalize_name(name) - other.discard(name) - target.add(name) - - -def fmt_ctl_formats(fmt_ctl, canonical_name): - result = {"binary", "source"} - if canonical_name in fmt_ctl.only_binary: - result.discard('source') - elif canonical_name in fmt_ctl.no_binary: - result.discard('binary') - elif ':all:' in fmt_ctl.only_binary: - result.discard('source') - elif ':all:' in fmt_ctl.no_binary: - result.discard('binary') - return frozenset(result) - - -def fmt_ctl_no_binary(fmt_ctl): - fmt_ctl_handle_mutual_exclude( - ':all:', fmt_ctl.no_binary, fmt_ctl.only_binary, - ) + yield Link(url, self.url, requires_python=pyrequire) Search = namedtuple('Search', 'supplied canonical formats') diff --git a/pipenv/patched/notpip/_internal/locations.py b/pipenv/patched/notpip/_internal/locations.py index 29c6db79..3c7d5bd8 100644 --- a/pipenv/patched/notpip/_internal/locations.py +++ b/pipenv/patched/notpip/_internal/locations.py @@ -10,8 +10,8 @@ import sysconfig from distutils import sysconfig as distutils_sysconfig from distutils.command.install import SCHEME_KEYS # type: ignore -from pipenv.patched.notpip._internal.compat import WINDOWS, expanduser from pipenv.patched.notpip._internal.utils import appdirs +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser # Application Directories USER_CACHE_DIR = appdirs.user_cache_dir("pip") diff --git a/pipenv/patched/notpip/_internal/models/candidate.py b/pipenv/patched/notpip/_internal/models/candidate.py new file mode 100644 index 00000000..9627589e --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/candidate.py @@ -0,0 +1,24 @@ +from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version + +from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin + + +class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + + def __init__(self, project, version, location, requires_python=None): + self.project = project + self.version = parse_version(version) + self.location = location + self.requires_python = requires_python + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), + defining_class=InstallationCandidate + ) + + def __repr__(self): + return "".format( + self.project, self.version, self.location, + ) diff --git a/pipenv/patched/notpip/_internal/models/format_control.py b/pipenv/patched/notpip/_internal/models/format_control.py new file mode 100644 index 00000000..caad3cba --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/format_control.py @@ -0,0 +1,62 @@ +from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name + + +class FormatControl(object): + """A helper class for controlling formats from which packages are installed. + If a field is falsy, it isn't set. If it is {':all:'}, it should match all + packages except those listed in the other field. Only one field can be set + to {':all:'} at a time. The rest of the time exact package name matches + are listed, with any given package only showing up in one field at a time. + """ + def __init__(self, no_binary=None, only_binary=None): + self.no_binary = set() if no_binary is None else no_binary + self.only_binary = set() if only_binary is None else only_binary + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "{}({}, {})".format( + self.__class__.__name__, + self.no_binary, + self.only_binary + ) + + @staticmethod + def handle_mutual_excludes(value, target, other): + new = value.split(',') + while ':all:' in new: + other.clear() + target.clear() + target.add(':all:') + del new[:new.index(':all:') + 1] + # Without a none, we want to discard everything as :all: covers it + if ':none:' not in new: + return + for name in new: + if name == ':none:': + target.clear() + continue + name = canonicalize_name(name) + other.discard(name) + target.add(name) + + def get_allowed_formats(self, canonical_name): + result = {"binary", "source"} + if canonical_name in self.only_binary: + result.discard('source') + elif canonical_name in self.no_binary: + result.discard('binary') + elif ':all:' in self.only_binary: + result.discard('source') + elif ':all:' in self.no_binary: + result.discard('binary') + return frozenset(result) + + def disallow_binaries(self): + self.handle_mutual_excludes( + ':all:', self.no_binary, self.only_binary, + ) diff --git a/pipenv/patched/notpip/_internal/models/index.py b/pipenv/patched/notpip/_internal/models/index.py index f9e84894..0983fc9c 100644 --- a/pipenv/patched/notpip/_internal/models/index.py +++ b/pipenv/patched/notpip/_internal/models/index.py @@ -1,15 +1,29 @@ from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -class Index(object): - def __init__(self, url): +class PackageIndex(object): + """Represents a Package Index and provides easier access to endpoints + """ + + def __init__(self, url, file_storage_domain): + super(PackageIndex, self).__init__() self.url = url self.netloc = urllib_parse.urlsplit(url).netloc - self.simple_url = self.url_to_path('simple') - self.pypi_url = self.url_to_path('pypi') + self.simple_url = self._url_for_path('simple') + self.pypi_url = self._url_for_path('pypi') - def url_to_path(self, path): + # This is part of a temporary hack used to block installs of PyPI + # packages which depend on external urls only necessary until PyPI can + # block such packages themselves + self.file_storage_domain = file_storage_domain + + def _url_for_path(self, path): return urllib_parse.urljoin(self.url, path) -PyPI = Index('https://pypi.org/') +PyPI = PackageIndex( + 'https://pypi.org/', file_storage_domain='files.pythonhosted.org' +) +TestPyPI = PackageIndex( + 'https://test.pypi.org/', file_storage_domain='test-files.pythonhosted.org' +) diff --git a/pipenv/patched/notpip/_internal/models/link.py b/pipenv/patched/notpip/_internal/models/link.py new file mode 100644 index 00000000..686af1d0 --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/link.py @@ -0,0 +1,141 @@ +import posixpath +import re + +from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse + +from pipenv.patched.notpip._internal.download import path_to_url +from pipenv.patched.notpip._internal.utils.misc import splitext +from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin +from pipenv.patched.notpip._internal.wheel import wheel_ext + + +class Link(KeyBasedCompareMixin): + """Represents a parsed link from a Package Index's simple URL + """ + + def __init__(self, url, comes_from=None, requires_python=None): + """ + url: + url of the resource pointed to (href of the link) + comes_from: + instance of HTMLPage where the link was found, or string. + requires_python: + String containing the `Requires-Python` metadata field, specified + in PEP 345. This may be specified by a data-requires-python + attribute in the HTML link tag, as described in PEP 503. + """ + + # url can be a UNC windows share + if url.startswith('\\\\'): + url = path_to_url(url) + + self.url = url + self.comes_from = comes_from + self.requires_python = requires_python if requires_python else None + + super(Link, self).__init__( + key=(self.url), + defining_class=Link + ) + + def __str__(self): + if self.requires_python: + rp = ' (requires-python:%s)' % self.requires_python + else: + rp = '' + if self.comes_from: + return '%s (from %s)%s' % (self.url, self.comes_from, rp) + else: + return str(self.url) + + def __repr__(self): + return '' % self + + @property + def filename(self): + _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) + name = posixpath.basename(path.rstrip('/')) or netloc + name = urllib_parse.unquote(name) + assert name, ('URL %r produced no filename' % self.url) + return name + + @property + def scheme(self): + return urllib_parse.urlsplit(self.url)[0] + + @property + def netloc(self): + return urllib_parse.urlsplit(self.url)[1] + + @property + def path(self): + return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) + + def splitext(self): + return splitext(posixpath.basename(self.path.rstrip('/'))) + + @property + def ext(self): + return self.splitext()[1] + + @property + def url_without_fragment(self): + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) + return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) + + _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') + + @property + def egg_fragment(self): + match = self._egg_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') + + @property + def subdirectory_fragment(self): + match = self._subdirectory_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _hash_re = re.compile( + r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' + ) + + @property + def hash(self): + match = self._hash_re.search(self.url) + if match: + return match.group(2) + return None + + @property + def hash_name(self): + match = self._hash_re.search(self.url) + if match: + return match.group(1) + return None + + @property + def show_url(self): + return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) + + @property + def is_wheel(self): + return self.ext == wheel_ext + + @property + def is_artifact(self): + """ + Determines if this points to an actual artifact (e.g. a tarball) or if + it points to an "abstract" thing like a path or a VCS location. + """ + from pipenv.patched.notpip._internal.vcs import vcs + + if self.scheme in vcs.all_schemes: + return False + + return True diff --git a/pipenv/patched/notpip/_internal/operations/freeze.py b/pipenv/patched/notpip/_internal/operations/freeze.py index 532989bc..b18b98e4 100644 --- a/pipenv/patched/notpip/_internal/operations/freeze.py +++ b/pipenv/patched/notpip/_internal/operations/freeze.py @@ -10,11 +10,13 @@ from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError from pipenv.patched.notpip._internal.exceptions import InstallationError -from pipenv.patched.notpip._internal.req import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, +) from pipenv.patched.notpip._internal.req.req_file import COMMENT_RE from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.misc import ( - dist_is_editable, get_installed_distributions, + dist_is_editable, get_installed_distributions, make_vcs_requirement_url, ) logger = logging.getLogger(__name__) @@ -99,13 +101,13 @@ def freeze( line = line[2:].strip() else: line = line[len('--editable'):].strip().lstrip('=') - line_req = InstallRequirement.from_editable( + line_req = install_req_from_editable( line, isolated=isolated, wheel_cache=wheel_cache, ) else: - line_req = InstallRequirement.from_line( + line_req = install_req_from_line( COMMENT_RE.sub('', line).strip(), isolated=isolated, wheel_cache=wheel_cache, @@ -166,7 +168,13 @@ class FrozenRequirement(object): _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') @classmethod - def from_dist(cls, dist, dependency_links): + def _init_args_from_dist(cls, dist, dependency_links): + """ + Compute and return arguments (req, editable, comments) to pass to + FrozenRequirement.__init__(). + + This method is for use in FrozenRequirement.from_dist(). + """ location = os.path.normcase(os.path.abspath(dist.location)) comments = [] from pipenv.patched.notpip._internal.vcs import vcs, get_src_requirement @@ -231,12 +239,15 @@ class FrozenRequirement(object): else: rev = '{%s}' % date_match.group(1) editable = True - req = '%s@%s#egg=%s' % ( - svn_location, - rev, - cls.egg_name(dist) - ) - return cls(dist.project_name, req, editable, comments) + egg_name = cls.egg_name(dist) + req = make_vcs_requirement_url(svn_location, rev, egg_name) + + return (req, editable, comments) + + @classmethod + def from_dist(cls, dist, dependency_links): + args = cls._init_args_from_dist(dist, dependency_links) + return cls(dist.project_name, *args) @staticmethod def egg_name(dist): diff --git a/pipenv/patched/notpip/_internal/operations/prepare.py b/pipenv/patched/notpip/_internal/operations/prepare.py index 9ebc3ebd..d61270b4 100644 --- a/pipenv/patched/notpip/_internal/operations/prepare.py +++ b/pipenv/patched/notpip/_internal/operations/prepare.py @@ -7,7 +7,6 @@ import os from pipenv.patched.notpip._vendor import pkg_resources, requests from pipenv.patched.notpip._internal.build_env import BuildEnvironment -from pipenv.patched.notpip._internal.compat import expanduser from pipenv.patched.notpip._internal.download import ( is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path, ) @@ -15,6 +14,7 @@ from pipenv.patched.notpip._internal.exceptions import ( DirectoryUrlHashUnsupported, HashUnpinned, InstallationError, PreviousBuildDirError, VcsHashUnsupported, ) +from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.hashes import MissingHashes from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import display_path, normalize_path, rmtree @@ -65,7 +65,7 @@ class DistAbstraction(object): """Return a setuptools Dist object.""" raise NotImplementedError(self.dist) - def prep_for_dist(self, finder): + def prep_for_dist(self, finder, build_isolation): """Ensure that we can get a Dist for this requirement.""" raise NotImplementedError(self.dist) @@ -93,36 +93,36 @@ class IsSDist(DistAbstraction): return dist def prep_for_dist(self, finder, build_isolation): - # Before calling "setup.py egg_info", we need to set-up the build - # environment. - build_requirements = self.req.get_pep_518_info() - should_isolate = build_isolation and build_requirements is not None + # Prepare for building. We need to: + # 1. Load pyproject.toml (if it exists) + # 2. Set up the build environment + + self.req.load_pyproject_toml() + should_isolate = self.req.use_pep517 and build_isolation if should_isolate: - # Haven't implemented PEP 517 yet, so spew a warning about it if - # build-requirements don't include setuptools and wheel. - missing_requirements = {'setuptools', 'wheel'} - { - pkg_resources.Requirement(r).key for r in build_requirements - } - if missing_requirements: + # Isolate in a BuildEnvironment and install the build-time + # requirements. + self.req.build_env = BuildEnvironment() + self.req.build_env.install_requirements( + finder, self.req.pyproject_requires, + "Installing build dependencies" + ) + missing = [] + if self.req.requirements_to_check: + check = self.req.requirements_to_check + missing = self.req.build_env.missing_requirements(check) + if missing: logger.warning( "Missing build requirements in pyproject.toml for %s.", self.req, ) logger.warning( - "This version of pip does not implement PEP 517 so it " - "cannot build a wheel without %s.", - " and ".join(map(repr, sorted(missing_requirements))) + "The project does not specify a build backend, and pip " + "cannot fall back to setuptools without %s.", + " and ".join(map(repr, sorted(missing))) ) - # Isolate in a BuildEnvironment and install the build-time - # requirements. - self.req.build_env = BuildEnvironment() - self.req.build_env.install_requirements( - finder, build_requirements, - "Installing build dependencies" - ) - try: self.req.run_egg_info() except (OSError, TypeError): @@ -136,7 +136,7 @@ class Installed(DistAbstraction): def dist(self, finder): return self.req.satisfied_by - def prep_for_dist(self, finder): + def prep_for_dist(self, finder, build_isolation): pass diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py index 4205f6e0..182c1c88 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py @@ -10,7 +10,12 @@ import sysconfig import warnings from collections import OrderedDict -import pipenv.patched.notpip._internal.utils.glibc +try: + import pipenv.patched.notpip._internal.utils.glibc +except ImportError: + import pipenv.patched.notpip.utils.glibc + +from pipenv.patched.notpip._internal.utils.compat import get_extension_suffixes logger = logging.getLogger(__name__) @@ -252,10 +257,9 @@ def get_supported(versions=None, noarch=False, platform=None, abis[0:0] = [abi] abi3s = set() - import imp - for suffix in imp.get_suffixes(): - if suffix[0].startswith('.abi'): - abi3s.add(suffix[0].split('.', 2)[1]) + for suffix in get_extension_suffixes(): + if suffix.startswith('.abi'): + abi3s.add(suffix.split('.', 2)[1]) abis.extend(sorted(list(abi3s))) diff --git a/pipenv/patched/notpip/_internal/pyproject.py b/pipenv/patched/notpip/_internal/pyproject.py new file mode 100644 index 00000000..a47e0f05 --- /dev/null +++ b/pipenv/patched/notpip/_internal/pyproject.py @@ -0,0 +1,144 @@ +from __future__ import absolute_import + +import io +import os + +from pipenv.patched.notpip._vendor import pytoml, six + +from pipenv.patched.notpip._internal.exceptions import InstallationError + + +def _is_list_of_str(obj): + return ( + isinstance(obj, list) and + all(isinstance(item, six.string_types) for item in obj) + ) + + +def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): + """Load the pyproject.toml file. + + Parameters: + use_pep517 - Has the user requested PEP 517 processing? None + means the user hasn't explicitly specified. + pyproject_toml - Location of the project's pyproject.toml file + setup_py - Location of the project's setup.py file + req_name - The name of the requirement we're processing (for + error reporting) + + Returns: + None if we should use the legacy code path, otherwise a tuple + ( + requirements from pyproject.toml, + name of PEP 517 backend, + requirements we should check are installed after setting + up the build environment + ) + """ + has_pyproject = os.path.isfile(pyproject_toml) + has_setup = os.path.isfile(setup_py) + + if has_pyproject: + with io.open(pyproject_toml, encoding="utf-8") as f: + pp_toml = pytoml.load(f) + build_system = pp_toml.get("build-system") + else: + build_system = None + + # The following cases must use PEP 517 + # We check for use_pep517 equalling False because that + # means the user explicitly requested --no-use-pep517 + if has_pyproject and not has_setup: + if use_pep517 is False: + raise InstallationError( + "Disabling PEP 517 processing is invalid: " + "project does not have a setup.py" + ) + use_pep517 = True + elif build_system and "build-backend" in build_system: + if use_pep517 is False: + raise InstallationError( + "Disabling PEP 517 processing is invalid: " + "project specifies a build backend of {} " + "in pyproject.toml".format( + build_system["build-backend"] + ) + ) + use_pep517 = True + + # If we haven't worked out whether to use PEP 517 yet, + # and the user hasn't explicitly stated a preference, + # we do so if the project has a pyproject.toml file. + elif use_pep517 is None: + use_pep517 = has_pyproject + + # At this point, we know whether we're going to use PEP 517. + assert use_pep517 is not None + + # If we're using the legacy code path, there is nothing further + # for us to do here. + if not use_pep517: + return None + + if build_system is None: + # Either the user has a pyproject.toml with no build-system + # section, or the user has no pyproject.toml, but has opted in + # explicitly via --use-pep517. + # In the absence of any explicit backend specification, we + # assume the setuptools backend, and require wheel and a version + # of setuptools that supports that backend. + build_system = { + "requires": ["setuptools>=38.2.5", "wheel"], + "build-backend": "setuptools.build_meta", + } + + # If we're using PEP 517, we have build system information (either + # from pyproject.toml, or defaulted by the code above). + # Note that at this point, we do not know if the user has actually + # specified a backend, though. + assert build_system is not None + + # Ensure that the build-system section in pyproject.toml conforms + # to PEP 518. + error_template = ( + "{package} has a pyproject.toml file that does not comply " + "with PEP 518: {reason}" + ) + + # Specifying the build-system table but not the requires key is invalid + if "requires" not in build_system: + raise InstallationError( + error_template.format(package=req_name, reason=( + "it has a 'build-system' table but not " + "'build-system.requires' which is mandatory in the table" + )) + ) + + # Error out if requires is not a list of strings + requires = build_system["requires"] + if not _is_list_of_str(requires): + raise InstallationError(error_template.format( + package=req_name, + reason="'build-system.requires' is not a list of strings.", + )) + + backend = build_system.get("build-backend") + check = [] + if backend is None: + # If the user didn't specify a backend, we assume they want to use + # the setuptools backend. But we can't be sure they have included + # a version of setuptools which supplies the backend, or wheel + # (which is neede by the backend) in their requirements. So we + # make a note to check that those requirements are present once + # we have set up the environment. + # TODO: Review this - it's quite a lot of work to check for a very + # specific case. The problem is, that case is potentially quite + # common - projects that adopted PEP 518 early for the ability to + # specify requirements to execute setup.py, but never considered + # needing to mention the build tools themselves. The original PEP + # 518 code had a similar check (but implemented in a different + # way). + backend = "setuptools.build_meta" + check = ["setuptools>=38.2.5", "wheel"] + + return (requires, backend, check) diff --git a/pipenv/patched/notpip/_internal/req/constructors.py b/pipenv/patched/notpip/_internal/req/constructors.py new file mode 100644 index 00000000..9fe28d88 --- /dev/null +++ b/pipenv/patched/notpip/_internal/req/constructors.py @@ -0,0 +1,298 @@ +"""Backing implementation for InstallRequirement's various constructors + +The idea here is that these formed a major chunk of InstallRequirement's size +so, moving them and support code dedicated to them outside of that class +helps creates for better understandability for the rest of the code. + +These are meant to be used elsewhere within pip to create instances of +InstallRequirement. +""" + +import logging +import os +import re +import traceback + +from pipenv.patched.notpip._vendor.packaging.markers import Marker +from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement +from pipenv.patched.notpip._vendor.packaging.specifiers import Specifier +from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError, parse_requirements + +from pipenv.patched.notpip._internal.download import ( + is_archive_file, is_url, path_to_url, url_to_path, +) +from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.models.index import PyPI, TestPyPI +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.utils.misc import is_installable_dir +from pipenv.patched.notpip._internal.vcs import vcs +from pipenv.patched.notpip._internal.wheel import Wheel + +__all__ = [ + "install_req_from_editable", "install_req_from_line", + "parse_editable" +] + +logger = logging.getLogger(__name__) +operators = Specifier._operators.keys() + + +def _strip_extras(path): + m = re.match(r'^(.+)(\[[^\]]+\])$', path) + extras = None + if m: + path_no_extras = m.group(1) + extras = m.group(2) + else: + path_no_extras = path + + return path_no_extras, extras + + +def parse_editable(editable_req): + """Parses an editable requirement into: + - a requirement name + - an URL + - extras + - editable options + Accepted requirements: + svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir + .[some_extra] + """ + + url = editable_req + + # If a file path is specified with extras, strip off the extras. + url_no_extras, extras = _strip_extras(url) + + if os.path.isdir(url_no_extras): + if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): + raise InstallationError( + "Directory %r is not installable. File 'setup.py' not found." % + url_no_extras + ) + # Treating it as code that has already been checked out + url_no_extras = path_to_url(url_no_extras) + + if url_no_extras.lower().startswith('file:'): + package_name = Link(url_no_extras).egg_fragment + if extras: + return ( + package_name, + url_no_extras, + Requirement("placeholder" + extras.lower()).extras, + ) + else: + return package_name, url_no_extras, None + + for version_control in vcs: + if url.lower().startswith('%s:' % version_control): + url = '%s+%s' % (version_control, url) + break + + if '+' not in url: + raise InstallationError( + '%s should either be a path to a local project or a VCS url ' + 'beginning with svn+, git+, hg+, or bzr+' % + editable_req + ) + + vc_type = url.split('+', 1)[0].lower() + + if not vcs.get_backend(vc_type): + error_message = 'For --editable=%s only ' % editable_req + \ + ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ + ' is currently supported' + raise InstallationError(error_message) + + package_name = Link(url).egg_fragment + if not package_name: + raise InstallationError( + "Could not detect requirement name for '%s', please specify one " + "with #egg=your_package_name" % editable_req + ) + return package_name, url, None + + +def deduce_helpful_msg(req): + """Returns helpful msg in case requirements file does not exist, + or cannot be parsed. + + :params req: Requirements file path + """ + msg = "" + if os.path.exists(req): + msg = " It does exist." + # Try to parse and check if it is a requirements file. + try: + with open(req, 'r') as fp: + # parse first line only + next(parse_requirements(fp.read())) + msg += " The argument you provided " + \ + "(%s) appears to be a" % (req) + \ + " requirements file. If that is the" + \ + " case, use the '-r' flag to install" + \ + " the packages specified within it." + except RequirementParseError: + logger.debug("Cannot parse '%s' as requirements \ + file" % (req), exc_info=1) + else: + msg += " File '%s' does not exist." % (req) + return msg + + +# ---- The actual constructors follow ---- + + +def install_req_from_editable( + editable_req, comes_from=None, isolated=False, options=None, + wheel_cache=None, constraint=False +): + name, url, extras_override = parse_editable(editable_req) + if url.startswith('file:'): + source_dir = url_to_path(url) + else: + source_dir = None + + if name is not None: + try: + req = Requirement(name) + except InvalidRequirement: + raise InstallationError("Invalid requirement: '%s'" % name) + else: + req = None + return InstallRequirement( + req, comes_from, source_dir=source_dir, + editable=True, + link=Link(url), + constraint=constraint, + isolated=isolated, + options=options if options else {}, + wheel_cache=wheel_cache, + extras=extras_override or (), + ) + + +def install_req_from_line( + name, comes_from=None, isolated=False, options=None, wheel_cache=None, + constraint=False +): + """Creates an InstallRequirement from a name, which might be a + requirement, directory containing 'setup.py', filename, or URL. + """ + if is_url(name): + marker_sep = '; ' + else: + marker_sep = ';' + if marker_sep in name: + name, markers = name.split(marker_sep, 1) + markers = markers.strip() + if not markers: + markers = None + else: + markers = Marker(markers) + else: + markers = None + name = name.strip() + req = None + path = os.path.normpath(os.path.abspath(name)) + link = None + extras = None + + if is_url(name): + link = Link(name) + else: + p, extras = _strip_extras(path) + looks_like_dir = os.path.isdir(p) and ( + os.path.sep in name or + (os.path.altsep is not None and os.path.altsep in name) or + name.startswith('.') + ) + if looks_like_dir: + if not is_installable_dir(p): + raise InstallationError( + "Directory %r is not installable. Neither 'setup.py' " + "nor 'pyproject.toml' found." % name + ) + link = Link(path_to_url(p)) + elif is_archive_file(p): + if not os.path.isfile(p): + logger.warning( + 'Requirement %r looks like a filename, but the ' + 'file does not exist', + name + ) + link = Link(path_to_url(p)) + + # it's a local file, dir, or url + if link: + # Handle relative file URLs + if link.scheme == 'file' and re.search(r'\.\./', link.url): + link = Link( + path_to_url(os.path.normpath(os.path.abspath(link.path)))) + # wheel file + if link.is_wheel: + wheel = Wheel(link.filename) # can raise InvalidWheelFilename + req = "%s==%s" % (wheel.name, wheel.version) + else: + # set the req to the egg fragment. when it's not there, this + # will become an 'unnamed' requirement + req = link.egg_fragment + + # a requirement specifier + else: + req = name + + if extras: + extras = Requirement("placeholder" + extras.lower()).extras + else: + extras = () + if req is not None: + try: + req = Requirement(req) + except InvalidRequirement: + if os.path.sep in req: + add_msg = "It looks like a path." + add_msg += deduce_helpful_msg(req) + elif '=' in req and not any(op in req for op in operators): + add_msg = "= is not a valid operator. Did you mean == ?" + else: + add_msg = traceback.format_exc() + raise InstallationError( + "Invalid requirement: '%s'\n%s" % (req, add_msg) + ) + + return InstallRequirement( + req, comes_from, link=link, markers=markers, + isolated=isolated, + options=options if options else {}, + wheel_cache=wheel_cache, + constraint=constraint, + extras=extras, + ) + + +def install_req_from_req( + req, comes_from=None, isolated=False, wheel_cache=None +): + try: + req = Requirement(req) + except InvalidRequirement: + raise InstallationError("Invalid requirement: '%s'" % req) + + domains_not_allowed = [ + PyPI.file_storage_domain, + TestPyPI.file_storage_domain, + ] + if req.url and comes_from.link.netloc in domains_not_allowed: + # Explicitly disallow pypi packages that depend on external urls + raise InstallationError( + "Packages installed from PyPI cannot depend on packages " + "which are not also hosted on PyPI.\n" + "%s depends on %s " % (comes_from.name, req) + ) + + return InstallRequirement( + req, comes_from, isolated=isolated, wheel_cache=wheel_cache + ) diff --git a/pipenv/patched/notpip/_internal/req/req_file.py b/pipenv/patched/notpip/_internal/req/req_file.py index 66a58022..5f23cd3a 100644 --- a/pipenv/patched/notpip/_internal/req/req_file.py +++ b/pipenv/patched/notpip/_internal/req/req_file.py @@ -13,10 +13,12 @@ import sys from pipenv.patched.notpip._vendor.six.moves import filterfalse from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._internal import cmdoptions +from pipenv.patched.notpip._internal.cli import cmdoptions from pipenv.patched.notpip._internal.download import get_file_content from pipenv.patched.notpip._internal.exceptions import RequirementsFileParseError -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, +) __all__ = ['parse_requirements'] @@ -151,7 +153,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, for dest in SUPPORTED_OPTIONS_REQ_DEST: if dest in opts.__dict__ and opts.__dict__[dest]: req_options[dest] = opts.__dict__[dest] - yield InstallRequirement.from_line( + yield install_req_from_line( args_str, line_comes_from, constraint=constraint, isolated=isolated, options=req_options, wheel_cache=wheel_cache ) @@ -159,7 +161,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, # yield an editable requirement elif opts.editables: isolated = options.isolated_mode if options else False - yield InstallRequirement.from_editable( + yield install_req_from_editable( opts.editables[0], comes_from=line_comes_from, constraint=constraint, isolated=isolated, wheel_cache=wheel_cache ) diff --git a/pipenv/patched/notpip/_internal/req/req_install.py b/pipenv/patched/notpip/_internal/req/req_install.py index a9c642c0..3f32892c 100644 --- a/pipenv/patched/notpip/_internal/req/req_install.py +++ b/pipenv/patched/notpip/_internal/req/req_install.py @@ -1,66 +1,46 @@ from __future__ import absolute_import -import io import logging import os -import re import shutil import sys import sysconfig -import traceback import zipfile from distutils.util import change_root -from email.parser import FeedParser # type: ignore -from pipenv.patched.notpip._vendor import pkg_resources, pytoml, six -from pipenv.patched.notpip._vendor.packaging import specifiers -from pipenv.patched.notpip._vendor.packaging.markers import Marker -from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement +from pipenv.patched.notpip._vendor import pkg_resources, six +from pipenv.patched.notpip._vendor.packaging.requirements import Requirement from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.packaging.version import Version from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version -from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError, parse_requirements +from pipenv.patched.notpip._vendor.pep517.wrappers import Pep517HookCaller from pipenv.patched.notpip._internal import wheel from pipenv.patched.notpip._internal.build_env import NoOpBuildEnvironment -from pipenv.patched.notpip._internal.compat import native_str -from pipenv.patched.notpip._internal.download import ( - is_archive_file, is_url, path_to_url, url_to_path, -) from pipenv.patched.notpip._internal.exceptions import InstallationError from pipenv.patched.notpip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.pyproject import load_pyproject_toml from pipenv.patched.notpip._internal.req.req_uninstall import UninstallPathSet +from pipenv.patched.notpip._internal.utils.compat import native_str from pipenv.patched.notpip._internal.utils.hashes import Hashes from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( _make_build_dir, ask_path_exists, backup_dir, call_subprocess, display_path, dist_in_site_packages, dist_in_usersite, ensure_dir, - get_installed_version, is_installable_dir, read_text_file, rmtree, + get_installed_version, rmtree, ) +from pipenv.patched.notpip._internal.utils.packaging import get_metadata from pipenv.patched.notpip._internal.utils.setuptools_build import SETUPTOOLS_SHIM from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.utils.ui import open_spinner from pipenv.patched.notpip._internal.vcs import vcs -from pipenv.patched.notpip._internal.wheel import Wheel, move_wheel_files +from pipenv.patched.notpip._internal.wheel import move_wheel_files logger = logging.getLogger(__name__) -operators = specifiers.Specifier._operators.keys() - - -def _strip_extras(path): - m = re.match(r'^(.+)(\[[^\]]+\])$', path) - extras = None - if m: - path_no_extras = m.group(1) - extras = m.group(2) - else: - path_no_extras = path - - return path_no_extras, extras - class InstallRequirement(object): """ @@ -87,7 +67,6 @@ class InstallRequirement(object): if link is not None: self.link = self.original_link = link else: - from pipenv.patched.notpip._internal.index import Link self.link = self.original_link = req and req.url and Link(req.url) if extras: @@ -128,155 +107,32 @@ class InstallRequirement(object): self.isolated = isolated self.build_env = NoOpBuildEnvironment() - # Constructors - # TODO: Move these out of this class into custom methods. - @classmethod - def from_editable(cls, editable_req, comes_from=None, isolated=False, - options=None, wheel_cache=None, constraint=False): - from pipenv.patched.notpip._internal.index import Link + # The static build requirements (from pyproject.toml) + self.pyproject_requires = None - name, url, extras_override = parse_editable(editable_req) - if url.startswith('file:'): - source_dir = url_to_path(url) - else: - source_dir = None + # Build requirements that we will check are available + # TODO: We don't do this for --no-build-isolation. Should we? + self.requirements_to_check = [] - if name is not None: - try: - req = Requirement(name) - except InvalidRequirement: - raise InstallationError("Invalid requirement: '%s'" % name) - else: - req = None - return cls( - req, comes_from, source_dir=source_dir, - editable=True, - link=Link(url), - constraint=constraint, - isolated=isolated, - options=options if options else {}, - wheel_cache=wheel_cache, - extras=extras_override or (), - ) + # The PEP 517 backend we should use to build the project + self.pep517_backend = None - @classmethod - def from_req(cls, req, comes_from=None, isolated=False, wheel_cache=None): - try: - req = Requirement(req) - except InvalidRequirement: - raise InstallationError("Invalid requirement: '%s'" % req) - if req.url: - raise InstallationError( - "Direct url requirement (like %s) are not allowed for " - "dependencies" % req - ) - return cls(req, comes_from, isolated=isolated, wheel_cache=wheel_cache) - - @classmethod - def from_line( - cls, name, comes_from=None, isolated=False, options=None, - wheel_cache=None, constraint=False): - """Creates an InstallRequirement from a name, which might be a - requirement, directory containing 'setup.py', filename, or URL. - """ - from pipenv.patched.notpip._internal.index import Link - - if is_url(name): - marker_sep = '; ' - else: - marker_sep = ';' - if marker_sep in name: - name, markers = name.split(marker_sep, 1) - markers = markers.strip() - if not markers: - markers = None - else: - markers = Marker(markers) - else: - markers = None - name = name.strip() - req = None - path = os.path.normpath(os.path.abspath(name)) - link = None - extras = None - - if is_url(name): - link = Link(name) - else: - p, extras = _strip_extras(path) - looks_like_dir = os.path.isdir(p) and ( - os.path.sep in name or - (os.path.altsep is not None and os.path.altsep in name) or - name.startswith('.') - ) - if looks_like_dir: - if not is_installable_dir(p): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' " - "not found." % name - ) - link = Link(path_to_url(p)) - elif is_archive_file(p): - if not os.path.isfile(p): - logger.warning( - 'Requirement %r looks like a filename, but the ' - 'file does not exist', - name - ) - link = Link(path_to_url(p)) - - # it's a local file, dir, or url - if link: - # Handle relative file URLs - if link.scheme == 'file' and re.search(r'\.\./', link.url): - link = Link( - path_to_url(os.path.normpath(os.path.abspath(link.path)))) - # wheel file - if link.is_wheel: - wheel = Wheel(link.filename) # can raise InvalidWheelFilename - req = "%s==%s" % (wheel.name, wheel.version) - else: - # set the req to the egg fragment. when it's not there, this - # will become an 'unnamed' requirement - req = link.egg_fragment - - # a requirement specifier - else: - req = name - - if extras: - extras = Requirement("placeholder" + extras.lower()).extras - else: - extras = () - if req is not None: - try: - req = Requirement(req) - except InvalidRequirement: - if os.path.sep in req: - add_msg = "It looks like a path." - add_msg += deduce_helpful_msg(req) - elif '=' in req and not any(op in req for op in operators): - add_msg = "= is not a valid operator. Did you mean == ?" - else: - add_msg = traceback.format_exc() - raise InstallationError( - "Invalid requirement: '%s'\n%s" % (req, add_msg)) - return cls( - req, comes_from, link=link, markers=markers, - isolated=isolated, - options=options if options else {}, - wheel_cache=wheel_cache, - constraint=constraint, - extras=extras, - ) + # Are we using PEP 517 for this requirement? + # After pyproject.toml has been loaded, the only valid values are True + # and False. Before loading, None is valid (meaning "use the default"). + # Setting an explicit value before loading pyproject.toml is supported, + # but after loading this flag should be treated as read only. + self.use_pep517 = None def __str__(self): if self.req: s = str(self.req) if self.link: s += ' from %s' % self.link.url + elif self.link: + s = self.link.url else: - s = self.link.url if self.link else None + s = '' if self.satisfied_by is not None: s += ' in %s' % display_path(self.satisfied_by.location) if self.comes_from: @@ -429,7 +285,7 @@ class InstallRequirement(object): package is not available until we run egg_info, so the build_location will return a temporary directory and store the _ideal_build_dir. - This is only called by self.egg_info_path to fix the temporary build + This is only called by self.run_egg_info to fix the temporary build directory. """ if self.source_dir is not None: @@ -557,48 +413,29 @@ class InstallRequirement(object): return pp_toml - def get_pep_518_info(self): - """Get PEP 518 build-time requirements. + def load_pyproject_toml(self): + """Load the pyproject.toml file. - Returns the list of the packages required to build the project, - specified as per PEP 518 within the package. If `pyproject.toml` is not - present, returns None to signify not using the same. + After calling this routine, all of the attributes related to PEP 517 + processing for this requirement have been set. In particular, the + use_pep517 attribute can be used to determine whether we should + follow the PEP 517 or legacy (setup.py) code path. """ - # If pyproject.toml does not exist, don't do anything. - if not os.path.isfile(self.pyproject_toml): - return None - - error_template = ( - "{package} has a pyproject.toml file that does not comply " - "with PEP 518: {reason}" + pep517_data = load_pyproject_toml( + self.use_pep517, + self.pyproject_toml, + self.setup_py, + str(self) ) - with io.open(self.pyproject_toml, encoding="utf-8") as f: - pp_toml = pytoml.load(f) - - # If there is no build-system table, just use setuptools and wheel. - if "build-system" not in pp_toml: - return ["setuptools", "wheel"] - - # Specifying the build-system table but not the requires key is invalid - build_system = pp_toml["build-system"] - if "requires" not in build_system: - raise InstallationError( - error_template.format(package=self, reason=( - "it has a 'build-system' table but not " - "'build-system.requires' which is mandatory in the table" - )) - ) - - # Error out if it's not a list of strings - requires = build_system["requires"] - if not _is_list_of_str(requires): - raise InstallationError(error_template.format( - package=self, - reason="'build-system.requires' is not a list of strings.", - )) - - return requires + if pep517_data is None: + self.use_pep517 = False + else: + self.use_pep517 = True + requires, backend, check = pep517_data + self.requirements_to_check = check + self.pyproject_requires = requires + self.pep517_backend = Pep517HookCaller(self.setup_py_dir, backend) def run_egg_info(self): assert self.source_dir @@ -615,7 +452,8 @@ class InstallRequirement(object): with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py - base_cmd = [os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', script] + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + base_cmd = [sys_executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] @@ -636,20 +474,20 @@ class InstallRequirement(object): command_desc='python setup.py egg_info') if not self.req: - if isinstance(parse_version(self.pkg_info()["Version"]), Version): + if isinstance(parse_version(self.metadata["Version"]), Version): op = "==" else: op = "===" self.req = Requirement( "".join([ - self.pkg_info()["Name"], + self.metadata["Name"], op, - self.pkg_info()["Version"], + self.metadata["Version"], ]) ) self._correct_build_location() else: - metadata_name = canonicalize_name(self.pkg_info()["Name"]) + metadata_name = canonicalize_name(self.metadata["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( 'Running setup.py (path:%s) egg_info for package %s ' @@ -659,19 +497,8 @@ class InstallRequirement(object): ) self.req = Requirement(metadata_name) - def egg_info_data(self, filename): - if self.satisfied_by is not None: - if not self.satisfied_by.has_metadata(filename): - return None - return self.satisfied_by.get_metadata(filename) - assert self.source_dir - filename = self.egg_info_path(filename) - if not os.path.exists(filename): - return None - data = read_text_file(filename) - return data - - def egg_info_path(self, filename): + @property + def egg_info_path(self): if self._egg_info_path is None: if self.editable: base = self.source_dir @@ -709,8 +536,7 @@ class InstallRequirement(object): if not filenames: raise InstallationError( - "Files/directories (from %s) not found in %s" - % (filename, base) + "Files/directories not found in %s" % base ) # if we have more than one match, we pick the toplevel one. This # can easily be the case if there is a dist folder which contains @@ -721,24 +547,18 @@ class InstallRequirement(object): (os.path.altsep and x.count(os.path.altsep) or 0) ) self._egg_info_path = os.path.join(base, filenames[0]) - return os.path.join(self._egg_info_path, filename) + return self._egg_info_path - def pkg_info(self): - p = FeedParser() - data = self.egg_info_data('PKG-INFO') - if not data: - logger.warning( - 'No PKG-INFO file found in %s', - display_path(self.egg_info_path('PKG-INFO')), - ) - p.feed(data or '') - return p.close() + @property + def metadata(self): + if not hasattr(self, '_metadata'): + self._metadata = get_metadata(self.get_dist()) - _requirements_section_re = re.compile(r'\[(.*?)\]') + return self._metadata def get_dist(self): """Return a pkg_resources.Distribution built from self.egg_info_path""" - egg_info = self.egg_info_path('').rstrip(os.path.sep) + egg_info = self.egg_info_path.rstrip(os.path.sep) base_dir = os.path.dirname(egg_info) metadata = pkg_resources.PathMetadata(base_dir, egg_info) dist_name = os.path.splitext(os.path.basename(egg_info))[0] @@ -750,7 +570,7 @@ class InstallRequirement(object): def assert_source_matches_version(self): assert self.source_dir - version = self.pkg_info()['version'] + version = self.metadata['version'] if self.req.specifier and version not in self.req.specifier: logger.warning( 'Requested %s, but installing version %s', @@ -794,10 +614,11 @@ class InstallRequirement(object): with indent_log(): # FIXME: should we do --install-headers here too? + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) with self.build_env: call_subprocess( [ - os.environ.get('PIP_PYTHON_PATH', sys.executable), + sys_executable, '-c', SETUPTOOLS_SHIM % self.setup_py ] + @@ -877,7 +698,7 @@ class InstallRequirement(object): def archive(self, build_dir): assert self.source_dir create_archive = True - archive_name = '%s-%s.zip' % (self.name, self.pkg_info()["version"]) + archive_name = '%s-%s.zip' % (self.name, self.metadata["version"]) archive_path = os.path.join(build_dir, archive_name) if os.path.exists(archive_path): response = ask_path_exists( @@ -1015,7 +836,8 @@ class InstallRequirement(object): def get_install_args(self, global_options, record_filename, root, prefix, pycompile): - install_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable), "-u"] + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + install_args = [sys_executable, "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ @@ -1039,104 +861,3 @@ class InstallRequirement(object): py_ver_str, self.name)] return install_args - - -def parse_editable(editable_req): - """Parses an editable requirement into: - - a requirement name - - an URL - - extras - - editable options - Accepted requirements: - svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir - .[some_extra] - """ - - from pipenv.patched.notpip._internal.index import Link - - url = editable_req - - # If a file path is specified with extras, strip off the extras. - url_no_extras, extras = _strip_extras(url) - - if os.path.isdir(url_no_extras): - if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' not found." % - url_no_extras - ) - # Treating it as code that has already been checked out - url_no_extras = path_to_url(url_no_extras) - - if url_no_extras.lower().startswith('file:'): - package_name = Link(url_no_extras).egg_fragment - if extras: - return ( - package_name, - url_no_extras, - Requirement("placeholder" + extras.lower()).extras, - ) - else: - return package_name, url_no_extras, None - - for version_control in vcs: - if url.lower().startswith('%s:' % version_control): - url = '%s+%s' % (version_control, url) - break - - if '+' not in url: - raise InstallationError( - '%s should either be a path to a local project or a VCS url ' - 'beginning with svn+, git+, hg+, or bzr+' % - editable_req - ) - - vc_type = url.split('+', 1)[0].lower() - - if not vcs.get_backend(vc_type): - error_message = 'For --editable=%s only ' % editable_req + \ - ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ - ' is currently supported' - raise InstallationError(error_message) - - package_name = Link(url).egg_fragment - if not package_name: - raise InstallationError( - "Could not detect requirement name for '%s', please specify one " - "with #egg=your_package_name" % editable_req - ) - return package_name, url, None - - -def deduce_helpful_msg(req): - """Returns helpful msg in case requirements file does not exist, - or cannot be parsed. - - :params req: Requirements file path - """ - msg = "" - if os.path.exists(req): - msg = " It does exist." - # Try to parse and check if it is a requirements file. - try: - with open(req, 'r') as fp: - # parse first line only - next(parse_requirements(fp.read())) - msg += " The argument you provided " + \ - "(%s) appears to be a" % (req) + \ - " requirements file. If that is the" + \ - " case, use the '-r' flag to install" + \ - " the packages specified within it." - except RequirementParseError: - logger.debug("Cannot parse '%s' as requirements \ - file" % (req), exc_info=1) - else: - msg += " File '%s' does not exist." % (req) - return msg - - -def _is_list_of_str(obj): - return ( - isinstance(obj, list) and - all(isinstance(item, six.string_types) for item in obj) - ) diff --git a/pipenv/patched/notpip/_internal/req/req_set.py b/pipenv/patched/notpip/_internal/req/req_set.py index 2c54c85a..a65851ff 100644 --- a/pipenv/patched/notpip/_internal/req/req_set.py +++ b/pipenv/patched/notpip/_internal/req/req_set.py @@ -12,19 +12,22 @@ logger = logging.getLogger(__name__) class RequirementSet(object): - def __init__(self, require_hashes=False, ignore_compatibility=True): + def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): """Create a RequirementSet. """ self.requirements = OrderedDict() self.require_hashes = require_hashes + self.check_supported_wheels = check_supported_wheels + if ignore_compatibility: + self.check_supported_wheels = False + self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False # Mapping of alias: real_name self.requirement_aliases = {} self.unnamed_requirements = [] self.successfully_downloaded = [] self.reqs_to_cleanup = [] - self.ignore_compatibility = ignore_compatibility def __str__(self): reqs = [req for req in self.requirements.values() @@ -56,17 +59,22 @@ class RequirementSet(object): requirement is applicable and has just been added. """ name = install_req.name + + # If the markers do not match, ignore this requirement. if not install_req.match_markers(extras_requested): - logger.info("Ignoring %s: markers '%s' don't match your " - "environment", install_req.name, - install_req.markers) + logger.info( + "Ignoring %s: markers '%s' don't match your environment", + name, install_req.markers, + ) return [], None - # This check has to come after we filter requirements with the - # environment markers. + # If the wheel is not supported, raise an error. + # Should check this after filtering out based on environment markers to + # allow specifying different wheels based on the environment/OS, in a + # single requirements file. if install_req.link and install_req.link.is_wheel: wheel = Wheel(install_req.link.filename) - if not wheel.supported() and not self.ignore_compatibility: + if self.check_supported_wheels and not wheel.supported(): raise InstallationError( "%s is not a supported wheel on this platform." % wheel.filename @@ -78,59 +86,73 @@ class RequirementSet(object): "a non direct req should have a parent" ) + # Unnamed requirements are scanned again and the requirement won't be + # added as a dependency until after scanning. if not name: # url or path requirement w/o an egg fragment self.unnamed_requirements.append(install_req) return [install_req], None - else: - try: - existing_req = self.get_requirement(name) - except KeyError: - existing_req = None - if (parent_req_name is None and existing_req and not - existing_req.constraint and - existing_req.extras == install_req.extras and not - existing_req.req.specifier == install_req.req.specifier): - raise InstallationError( - 'Double requirement given: %s (already in %s, name=%r)' - % (install_req, existing_req, name)) - if not existing_req: - # Add requirement - self.requirements[name] = install_req - # FIXME: what about other normalizations? E.g., _ vs. -? - if name.lower() != name: - self.requirement_aliases[name.lower()] = name - result = [install_req] - else: - # Assume there's no need to scan, and that we've already - # encountered this for scanning. - result = [] - if not install_req.constraint and existing_req.constraint: - if (install_req.link and not (existing_req.link and - install_req.link.path == existing_req.link.path)): - self.reqs_to_cleanup.append(install_req) - raise InstallationError( - "Could not satisfy constraints for '%s': " - "installation from path or url cannot be " - "constrained to a version" % name, - ) - # If we're now installing a constraint, mark the existing - # object for real installation. - existing_req.constraint = False - existing_req.extras = tuple( - sorted(set(existing_req.extras).union( - set(install_req.extras)))) - logger.debug("Setting %s extras to: %s", - existing_req, existing_req.extras) - # And now we need to scan this. - result = [existing_req] - # Canonicalise to the already-added object for the backref - # check below. - install_req = existing_req - # We return install_req here to allow for the caller to add it to - # the dependency information for the parent package. - return result, install_req + try: + existing_req = self.get_requirement(name) + except KeyError: + existing_req = None + + has_conflicting_requirement = ( + parent_req_name is None and + existing_req and + not existing_req.constraint and + existing_req.extras == install_req.extras and + existing_req.req.specifier != install_req.req.specifier + ) + if has_conflicting_requirement: + raise InstallationError( + "Double requirement given: %s (already in %s, name=%r)" + % (install_req, existing_req, name) + ) + + # When no existing requirement exists, add the requirement as a + # dependency and it will be scanned again after. + if not existing_req: + self.requirements[name] = install_req + # FIXME: what about other normalizations? E.g., _ vs. -? + if name.lower() != name: + self.requirement_aliases[name.lower()] = name + # We'd want to rescan this requirements later + return [install_req], install_req + + # Assume there's no need to scan, and that we've already + # encountered this for scanning. + if install_req.constraint or not existing_req.constraint: + return [], existing_req + + does_not_satisfy_constraint = ( + install_req.link and + not ( + existing_req.link and + install_req.link.path == existing_req.link.path + ) + ) + if does_not_satisfy_constraint: + self.reqs_to_cleanup.append(install_req) + raise InstallationError( + "Could not satisfy constraints for '%s': " + "installation from path or url cannot be " + "constrained to a version" % name, + ) + # If we're now installing a constraint, mark the existing + # object for real installation. + existing_req.constraint = False + existing_req.extras = tuple(sorted( + set(existing_req.extras) | set(install_req.extras) + )) + logger.debug( + "Setting %s extras to: %s", + existing_req, existing_req.extras, + ) + # Return the existing requirement for addition to the parent and + # scanning again. + return [existing_req], existing_req def has_requirement(self, project_name): name = project_name.lower() @@ -152,7 +174,7 @@ class RequirementSet(object): return self.requirements[name] if name in self.requirement_aliases: return self.requirements[self.requirement_aliases[name]] - # raise KeyError("No project with the name %r" % project_name) + pass def cleanup_files(self): """Clean up files, remove builds.""" diff --git a/pipenv/patched/notpip/_internal/req/req_uninstall.py b/pipenv/patched/notpip/_internal/req/req_uninstall.py index 3ccd3265..4cd15d84 100644 --- a/pipenv/patched/notpip/_internal/req/req_uninstall.py +++ b/pipenv/patched/notpip/_internal/req/req_uninstall.py @@ -9,9 +9,9 @@ import sysconfig from pipenv.patched.notpip._vendor import pkg_resources -from pipenv.patched.notpip._internal.compat import WINDOWS, cache_from_source, uses_pycache from pipenv.patched.notpip._internal.exceptions import UninstallationError from pipenv.patched.notpip._internal.locations import bin_py, bin_user +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, cache_from_source, uses_pycache from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local, @@ -120,6 +120,8 @@ def compress_for_output_listing(paths): folders.add(os.path.dirname(path)) files.add(path) + _normcased_files = set(map(os.path.normcase, files)) + folders = compact(folders) # This walks the tree using os.walk to not miss extra folders @@ -130,8 +132,9 @@ def compress_for_output_listing(paths): if fname.endswith(".pyc"): continue - file_ = os.path.normcase(os.path.join(dirpath, fname)) - if os.path.isfile(file_) and file_ not in files: + file_ = os.path.join(dirpath, fname) + if (os.path.isfile(file_) and + os.path.normcase(file_) not in _normcased_files): # We are skipping this file. Add it to the set. will_skip.add(file_) diff --git a/pipenv/patched/notpip/_internal/resolve.py b/pipenv/patched/notpip/_internal/resolve.py index 461a2bb8..b0d096f9 100644 --- a/pipenv/patched/notpip/_internal/resolve.py +++ b/pipenv/patched/notpip/_internal/resolve.py @@ -18,7 +18,7 @@ from pipenv.patched.notpip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors, UnsupportedPythonVersion, ) -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import install_req_from_req from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import dist_in_usersite, ensure_dir from pipenv.patched.notpip._internal.utils.packaging import check_dist_requires_python @@ -249,9 +249,6 @@ class Resolver(object): # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) - if ignore_requires_python or self.ignore_requires_python: - self.ignore_compatibility = True - if req_to_install.constraint or req_to_install.prepared: return [] @@ -267,7 +264,7 @@ class Resolver(object): try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: - if self.ignore_compatibility: + if self.ignore_requires_python or self.ignore_compatibility: logger.warning(err.args[0]) else: raise @@ -281,7 +278,7 @@ class Resolver(object): more_reqs = [] def add_req(subreq, extras_requested): - sub_install_req = InstallRequirement.from_req( + sub_install_req = install_req_from_req( str(subreq), req_to_install, isolated=self.isolated, @@ -303,10 +300,10 @@ class Resolver(object): # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): + # 'unnamed' requirements will get added here available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) - # 'unnamed' requirements will get added here req_to_install.is_direct = True requirement_set.add_requirement( req_to_install, parent_req_name=None, @@ -338,7 +335,7 @@ class Resolver(object): for available in available_requested: if hasattr(dist, '_DistInfoDistribution__dep_map'): for req in dist._DistInfoDistribution__dep_map[available]: - req = InstallRequirement.from_req( + req = install_req_from_req( str(req), req_to_install, isolated=self.isolated, diff --git a/pipenv/patched/notpip/_internal/utils/appdirs.py b/pipenv/patched/notpip/_internal/utils/appdirs.py index 291de7a9..e8e14526 100644 --- a/pipenv/patched/notpip/_internal/utils/appdirs.py +++ b/pipenv/patched/notpip/_internal/utils/appdirs.py @@ -9,7 +9,7 @@ import sys from pipenv.patched.notpip._vendor.six import PY2, text_type -from pipenv.patched.notpip._internal.compat import WINDOWS, expanduser +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser def user_cache_dir(appname): diff --git a/pipenv/patched/notpip/_internal/compat.py b/pipenv/patched/notpip/_internal/utils/compat.py similarity index 96% rename from pipenv/patched/notpip/_internal/compat.py rename to pipenv/patched/notpip/_internal/utils/compat.py index 6e51e32a..483bfdc8 100644 --- a/pipenv/patched/notpip/_internal/compat.py +++ b/pipenv/patched/notpip/_internal/utils/compat.py @@ -25,6 +25,7 @@ except ImportError: __all__ = [ "ipaddress", "uses_pycache", "console_to_str", "native_str", "get_path_uid", "stdlib_pkgs", "WINDOWS", "samefile", "get_terminal_size", + "get_extension_suffixes", ] @@ -160,6 +161,18 @@ def get_path_uid(path): return file_uid +if sys.version_info >= (3, 4): + from importlib.machinery import EXTENSION_SUFFIXES + + def get_extension_suffixes(): + return EXTENSION_SUFFIXES +else: + from imp import get_suffixes + + def get_extension_suffixes(): + return [suffix[0] for suffix in get_suffixes()] + + def expanduser(path): """ Expand ~ and ~user constructions. diff --git a/pipenv/patched/notpip/_internal/utils/filesystem.py b/pipenv/patched/notpip/_internal/utils/filesystem.py index 91976486..e8d6a2bb 100644 --- a/pipenv/patched/notpip/_internal/utils/filesystem.py +++ b/pipenv/patched/notpip/_internal/utils/filesystem.py @@ -1,7 +1,7 @@ import os import os.path -from pipenv.patched.notpip._internal.compat import get_path_uid +from pipenv.patched.notpip._internal.utils.compat import get_path_uid def check_path_owner(path): diff --git a/pipenv/patched/notpip/_internal/utils/logging.py b/pipenv/patched/notpip/_internal/utils/logging.py index 257a6234..576c4fa0 100644 --- a/pipenv/patched/notpip/_internal/utils/logging.py +++ b/pipenv/patched/notpip/_internal/utils/logging.py @@ -5,7 +5,7 @@ import logging import logging.handlers import os -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.misc import ensure_dir try: diff --git a/pipenv/patched/notpip/_internal/utils/misc.py b/pipenv/patched/notpip/_internal/utils/misc.py index e254f3d4..45e5204c 100644 --- a/pipenv/patched/notpip/_internal/utils/misc.py +++ b/pipenv/patched/notpip/_internal/utils/misc.py @@ -26,14 +26,14 @@ from pipenv.patched.notpip._vendor.six import PY2 from pipenv.patched.notpip._vendor.six.moves import input from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._internal.compat import ( - WINDOWS, console_to_str, expanduser, stdlib_pkgs, -) from pipenv.patched.notpip._internal.exceptions import CommandError, InstallationError from pipenv.patched.notpip._internal.locations import ( running_under_virtualenv, site_packages, user_site, virtualenv_no_global, write_delete_marker_file, ) +from pipenv.patched.notpip._internal.utils.compat import ( + WINDOWS, console_to_str, expanduser, stdlib_pkgs, +) if PY2: from io import BytesIO as StringIO @@ -96,7 +96,7 @@ def get_prog(): try: prog = os.path.basename(sys.argv[0]) if prog in ('__main__.py', '-c'): - return "%s -m pip" % os.environ.get('PIP_PYTHON_PATH', sys.executable) + return "%s -m pip" % sys.executable else: return prog except (AttributeError, TypeError, IndexError): @@ -187,12 +187,16 @@ def format_size(bytes): def is_installable_dir(path): - """Return True if `path` is a directory containing a setup.py file.""" + """Is path is a directory containing setup.py or pyproject.toml? + """ if not os.path.isdir(path): return False setup_py = os.path.join(path, 'setup.py') if os.path.isfile(setup_py): return True + pyproject_toml = os.path.join(path, 'pyproject.toml') + if os.path.isfile(pyproject_toml): + return True return False @@ -852,6 +856,44 @@ def enum(*sequential, **named): return type('Enum', (), enums) +def make_vcs_requirement_url(repo_url, rev, egg_project_name, subdir=None): + """ + Return the URL for a VCS requirement. + + Args: + repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + """ + req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) + if subdir: + req += '&subdirectory={}'.format(subdir) + + return req + + +def split_auth_from_netloc(netloc): + """ + Parse out and remove the auth information from a netloc. + + Returns: (netloc, (username, password)). + """ + if '@' not in netloc: + return netloc, (None, None) + + # Split from the right because that's how urllib.parse.urlsplit() + # behaves if more than one @ is present (which can be checked using + # the password attribute of urlsplit()'s return value). + auth, netloc = netloc.rsplit('@', 1) + if ':' in auth: + # Split from the left because that's how urllib.parse.urlsplit() + # behaves if more than one : is present (which again can be checked + # using the password attribute of the return value) + user_pass = tuple(auth.split(':', 1)) + else: + user_pass = auth, None + + return netloc, user_pass + + def remove_auth_from_url(url): # Return a copy of url with 'username:password@' removed. # username/pass params are passed to subversion through flags @@ -859,12 +901,11 @@ def remove_auth_from_url(url): # parsed url purl = urllib_parse.urlsplit(url) - stripped_netloc = \ - purl.netloc.split('@')[-1] + netloc, user_pass = split_auth_from_netloc(purl.netloc) # stripped url url_pieces = ( - purl.scheme, stripped_netloc, purl.path, purl.query, purl.fragment + purl.scheme, netloc, purl.path, purl.query, purl.fragment ) surl = urllib_parse.urlunsplit(url_pieces) return surl diff --git a/pipenv/patched/notpip/_internal/utils/models.py b/pipenv/patched/notpip/_internal/utils/models.py new file mode 100644 index 00000000..d5cb80a7 --- /dev/null +++ b/pipenv/patched/notpip/_internal/utils/models.py @@ -0,0 +1,40 @@ +"""Utilities for defining models +""" + +import operator + + +class KeyBasedCompareMixin(object): + """Provides comparision capabilities that is based on a key + """ + + def __init__(self, key, defining_class): + self._compare_key = key + self._defining_class = defining_class + + def __hash__(self): + return hash(self._compare_key) + + def __lt__(self, other): + return self._compare(other, operator.__lt__) + + def __le__(self, other): + return self._compare(other, operator.__le__) + + def __gt__(self, other): + return self._compare(other, operator.__gt__) + + def __ge__(self, other): + return self._compare(other, operator.__ge__) + + def __eq__(self, other): + return self._compare(other, operator.__eq__) + + def __ne__(self, other): + return self._compare(other, operator.__ne__) + + def _compare(self, other, method): + if not isinstance(other, self._defining_class): + return NotImplemented + + return method(self._compare_key, other._compare_key) diff --git a/pipenv/patched/notpip/_internal/utils/outdated.py b/pipenv/patched/notpip/_internal/utils/outdated.py index 6133e6fd..f8b1fe04 100644 --- a/pipenv/patched/notpip/_internal/utils/outdated.py +++ b/pipenv/patched/notpip/_internal/utils/outdated.py @@ -9,8 +9,8 @@ import sys from pipenv.patched.notpip._vendor import lockfile, pkg_resources from pipenv.patched.notpip._vendor.packaging import version as packaging_version -from pipenv.patched.notpip._internal.compat import WINDOWS from pipenv.patched.notpip._internal.index import PackageFinder +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner from pipenv.patched.notpip._internal.utils.misc import ensure_dir, get_installed_version @@ -22,16 +22,25 @@ logger = logging.getLogger(__name__) class SelfCheckState(object): def __init__(self, cache_dir): - self.statefile_path = os.path.join(cache_dir, "selfcheck.json") + self.state = {} + self.statefile_path = None - # Load the existing state - try: - with open(self.statefile_path) as statefile: - self.state = json.load(statefile)[sys.prefix] - except (IOError, ValueError, KeyError): - self.state = {} + # Try to load the existing state + if cache_dir: + self.statefile_path = os.path.join(cache_dir, "selfcheck.json") + try: + with open(self.statefile_path) as statefile: + self.state = json.load(statefile)[sys.prefix] + except (IOError, ValueError, KeyError): + # Explicitly suppressing exceptions, since we don't want to + # error out if the cache file is invalid. + pass def save(self, pypi_version, current_time): + # If we do not have a path to cache in, don't bother saving. + if not self.statefile_path: + return + # Check to make sure that we own the directory if not check_path_owner(os.path.dirname(self.statefile_path)): return diff --git a/pipenv/patched/notpip/_internal/utils/packaging.py b/pipenv/patched/notpip/_internal/utils/packaging.py index 13547743..d1e8ecaa 100644 --- a/pipenv/patched/notpip/_internal/utils/packaging.py +++ b/pipenv/patched/notpip/_internal/utils/packaging.py @@ -8,6 +8,7 @@ from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.packaging import specifiers, version from pipenv.patched.notpip._internal import exceptions +from pipenv.patched.notpip._internal.utils.misc import display_path logger = logging.getLogger(__name__) @@ -35,22 +36,31 @@ def check_requires_python(requires_python): def get_metadata(dist): if (isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata('METADATA')): - return dist.get_metadata('METADATA') + metadata = dist.get_metadata('METADATA') elif dist.has_metadata('PKG-INFO'): - return dist.get_metadata('PKG-INFO') + metadata = dist.get_metadata('PKG-INFO') + else: + logger.warning("No metadata found in %s", display_path(dist.location)) + metadata = '' + + feed_parser = FeedParser() + feed_parser.feed(metadata) + return feed_parser.close() def check_dist_requires_python(dist, absorb=True): - metadata = get_metadata(dist) - feed_parser = FeedParser() - feed_parser.feed(metadata) - pkg_info_dict = feed_parser.close() + pkg_info_dict = get_metadata(dist) requires_python = pkg_info_dict.get('Requires-Python') - if not absorb: + if absorb: return requires_python try: if not check_requires_python(requires_python): - return requires_python + raise exceptions.UnsupportedPythonVersion( + "%s requires Python '%s' but the running Python is %s" % ( + dist.project_name, + requires_python, + '.'.join(map(str, sys.version_info[:3])),) + ) except specifiers.InvalidSpecifier as e: logger.warning( "Package %s has an invalid Requires-Python entry %s - %s", diff --git a/pipenv/patched/notpip/_internal/utils/ui.py b/pipenv/patched/notpip/_internal/utils/ui.py index b96863cc..6eebd17d 100644 --- a/pipenv/patched/notpip/_internal/utils/ui.py +++ b/pipenv/patched/notpip/_internal/utils/ui.py @@ -15,7 +15,7 @@ from pipenv.patched.notpip._vendor.progress.bar import ( from pipenv.patched.notpip._vendor.progress.helpers import HIDE_CURSOR, SHOW_CURSOR, WritelnMixin from pipenv.patched.notpip._vendor.progress.spinner import Spinner -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.logging import get_indentation from pipenv.patched.notpip._internal.utils.misc import format_size from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/pipenv/patched/notpip/_internal/vcs/__init__.py b/pipenv/patched/notpip/_internal/vcs/__init__.py index 146f2829..5aeac633 100644 --- a/pipenv/patched/notpip/_internal/vcs/__init__.py +++ b/pipenv/patched/notpip/_internal/vcs/__init__.py @@ -17,7 +17,7 @@ from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Dict, Optional, Tuple # noqa: F401 - from pipenv.patched.notpip._internal.basecommand import Command # noqa: F401 + from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 __all__ = ['vcs', 'get_src_requirement'] @@ -200,12 +200,6 @@ class VersionControl(object): drive, tail = os.path.splitdrive(repo) return repo.startswith(os.path.sep) or drive - # See issue #1083 for why this method was introduced: - # https://github.com/pypa/pip/issues/1083 - def translate_egg_surname(self, surname): - # For example, Django has branches of the form "stable/1.7.x". - return surname.replace('/', '_') - def export(self, location): """ Export the repository at the url to the destination location @@ -213,51 +207,65 @@ class VersionControl(object): """ raise NotImplementedError - def get_url_rev(self, url): + def get_netloc_and_auth(self, netloc, scheme): """ - Returns the correct repository URL and revision by parsing the given - repository URL + Parse the repository URL's netloc, and return the new netloc to use + along with auth information. + + Args: + netloc: the original repository URL netloc. + scheme: the repository URL's scheme without the vcs prefix. + + This is mainly for the Subversion class to override, so that auth + information can be provided via the --username and --password options + instead of through the URL. For other subclasses like Git without + such an option, auth information must stay in the URL. + + Returns: (netloc, (username, password)). + """ + return netloc, (None, None) + + def get_url_rev_and_auth(self, url): + """ + Parse the repository URL to use, and return the URL, revision, + and auth info to use. + + Returns: (url, rev, (username, password)). """ - error_message = ( - "Sorry, '%s' is a malformed VCS url. " - "The format is +://, " - "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp" - ) - assert '+' in url, error_message % url - url = url.split('+', 1)[1] scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) + if '+' not in scheme: + raise ValueError( + "Sorry, {!r} is a malformed VCS url. " + "The format is +://, " + "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) + ) + # Remove the vcs prefix. + scheme = scheme.split('+', 1)[1] + netloc, user_pass = self.get_netloc_and_auth(netloc, scheme) rev = None if '@' in path: path, rev = path.rsplit('@', 1) url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev + return url, rev, user_pass - def get_url_rev_args(self, url): + def make_rev_args(self, username, password): """ - Return the URL and RevOptions "extra arguments" to use in obtain(), - as a tuple (url, extra_args). + Return the RevOptions "extra arguments" to use in obtain(). """ - return url, [] + return [] def get_url_rev_options(self, url): """ Return the URL and RevOptions object to use in obtain() and in some cases export(), as a tuple (url, rev_options). """ - url, rev = self.get_url_rev(url) - url, extra_args = self.get_url_rev_args(url) + url, rev, user_pass = self.get_url_rev_and_auth(url) + username, password = user_pass + extra_args = self.make_rev_args(username, password) rev_options = self.make_rev_options(rev, extra_args=extra_args) return url, rev_options - def get_info(self, location): - """ - Returns (url, revision), where both are strings - """ - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - return self.get_url(location), self.get_revision(location) - def normalize_url(self, url): """ Normalize a URL for comparison by unquoting it and removing any @@ -291,7 +299,7 @@ class VersionControl(object): """ raise NotImplementedError - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): """ Update an already-existing repo to the given ``rev_options``. @@ -341,7 +349,7 @@ class VersionControl(object): self.repo_name, rev_display, ) - self.update(dest, rev_options) + self.update(dest, url, rev_options) else: logger.info('Skipping because already up-to-date.') return @@ -421,8 +429,6 @@ class VersionControl(object): def get_url(self, location): """ Return the url used at location - - This is used in get_info() and obtain(). """ raise NotImplementedError diff --git a/pipenv/patched/notpip/_internal/vcs/bazaar.py b/pipenv/patched/notpip/_internal/vcs/bazaar.py index b2664cd8..890448ed 100644 --- a/pipenv/patched/notpip/_internal/vcs/bazaar.py +++ b/pipenv/patched/notpip/_internal/vcs/bazaar.py @@ -6,7 +6,9 @@ import os from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import display_path, rmtree +from pipenv.patched.notpip._internal.utils.misc import ( + display_path, make_vcs_requirement_url, rmtree, +) from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -62,16 +64,16 @@ class Bazaar(VersionControl): def switch(self, dest, url, rev_options): self.run_command(['switch', url], cwd=dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): cmd_args = ['pull', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it - url, rev = super(Bazaar, self).get_url_rev(url) + url, rev, user_pass = super(Bazaar, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'bzr+' + url - return url, rev + return url, rev, user_pass def get_url(self, location): urls = self.run_command(['info'], show_stdout=False, cwd=location) @@ -98,9 +100,9 @@ class Bazaar(VersionControl): return None if not repo.lower().startswith('bzr:'): repo = 'bzr+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] current_rev = self.get_revision(location) - return '%s@%s#egg=%s' % (repo, current_rev, egg_project_name) + egg_project_name = dist.egg_name().split('-', 1)[0] + return make_vcs_requirement_url(repo, current_rev, egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/git.py b/pipenv/patched/notpip/_internal/vcs/git.py index ef2dd908..3db56144 100644 --- a/pipenv/patched/notpip/_internal/vcs/git.py +++ b/pipenv/patched/notpip/_internal/vcs/git.py @@ -8,9 +8,9 @@ from pipenv.patched.notpip._vendor.packaging.version import parse as parse_versi from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._internal.compat import samefile from pipenv.patched.notpip._internal.exceptions import BadCommand -from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.compat import samefile +from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -77,6 +77,20 @@ class Git(VersionControl): version = '.'.join(version.split('.')[:3]) return parse_version(version) + def get_branch(self, location): + """ + Return the current branch, or None if HEAD isn't at a branch + (e.g. detached HEAD). + """ + args = ['rev-parse', '--abbrev-ref', 'HEAD'] + output = self.run_command(args, show_stdout=False, cwd=location) + branch = output.strip() + + if branch == 'HEAD': + return None + + return branch + def export(self, location): """Export the Git repository at the url to the destination location""" if not location.endswith('/'): @@ -91,8 +105,8 @@ class Git(VersionControl): def get_revision_sha(self, dest, rev): """ - Return a commit hash for the given revision if it names a remote - branch or tag. Otherwise, return None. + Return (sha_or_none, is_branch), where sha_or_none is a commit hash + if the revision names a remote branch or tag, otherwise None. Args: dest: the repository directory. @@ -115,22 +129,30 @@ class Git(VersionControl): branch_ref = 'refs/remotes/origin/{}'.format(rev) tag_ref = 'refs/tags/{}'.format(rev) - return refs.get(branch_ref) or refs.get(tag_ref) + sha = refs.get(branch_ref) + if sha is not None: + return (sha, True) - def check_rev_options(self, dest, rev_options): - """Check the revision options before checkout. + sha = refs.get(tag_ref) - Returns a new RevOptions object for the SHA1 of the branch or tag - if found. + return (sha, False) + + def resolve_revision(self, dest, url, rev_options): + """ + Resolve a revision to a new RevOptions object with the SHA1 of the + branch, tag, or ref if found. Args: rev_options: a RevOptions object. """ rev = rev_options.arg_rev - sha = self.get_revision_sha(dest, rev) + sha, is_branch = self.get_revision_sha(dest, rev) if sha is not None: - return rev_options.make_new(sha) + rev_options = rev_options.make_new(sha) + rev_options.branch_name = rev if is_branch else None + + return rev_options # Do not show a warning for the common case of something that has # the form of a Git commit hash. @@ -139,6 +161,19 @@ class Git(VersionControl): "Did not find branch or tag '%s', assuming revision or ref.", rev, ) + + if not rev.startswith('refs/'): + return rev_options + + # If it looks like a ref, we have to fetch it explicitly. + self.run_command( + ['fetch', '-q', url] + rev_options.to_args(), + cwd=dest, + ) + # Change the revision to the SHA of the ref we fetched + sha = self.get_revision(dest, rev='FETCH_HEAD') + rev_options = rev_options.make_new(sha) + return rev_options def is_commit_id_equal(self, dest, name): @@ -164,20 +199,22 @@ class Git(VersionControl): if rev_options.rev: # Then a specific revision was requested. - rev_options = self.check_rev_options(dest, rev_options) - # Only do a checkout if the current commit id doesn't match - # the requested revision. - if not self.is_commit_id_equal(dest, rev_options.rev): - rev = rev_options.rev - # Only fetch the revision if it's a ref - if rev.startswith('refs/'): - self.run_command( - ['fetch', '-q', url] + rev_options.to_args(), - cwd=dest, - ) - # Change the revision to the SHA of the ref we fetched - rev = 'FETCH_HEAD' - self.run_command(['checkout', '-q', rev], cwd=dest) + rev_options = self.resolve_revision(dest, url, rev_options) + branch_name = getattr(rev_options, 'branch_name', None) + if branch_name is None: + # Only do a checkout if the current commit id doesn't match + # the requested revision. + if not self.is_commit_id_equal(dest, rev_options.rev): + cmd_args = ['checkout', '-q'] + rev_options.to_args() + self.run_command(cmd_args, cwd=dest) + elif self.get_branch(dest) != branch_name: + # Then a specific branch was requested, and that branch + # is not yet checked out. + track_branch = 'origin/{}'.format(branch_name) + cmd_args = [ + 'checkout', '-b', branch_name, '--track', track_branch, + ] + self.run_command(cmd_args, cwd=dest) #: repo may contain submodules self.update_submodules(dest) @@ -189,7 +226,7 @@ class Git(VersionControl): self.update_submodules(dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): # First fetch changes from the default remote if self.get_git_version() >= parse_version('1.9.0'): # fetch tags in addition to everything else @@ -197,7 +234,7 @@ class Git(VersionControl): else: self.run_command(['fetch', '-q'], cwd=dest) # Then reset to wanted revision (maybe even origin/master) - rev_options = self.check_rev_options(dest, rev_options) + rev_options = self.resolve_revision(dest, url, rev_options) cmd_args = ['reset', '--hard', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) #: update submodules @@ -218,9 +255,11 @@ class Git(VersionControl): url = found_remote.split(' ')[1] return url.strip() - def get_revision(self, location): + def get_revision(self, location, rev=None): + if rev is None: + rev = 'HEAD' current_rev = self.run_command( - ['rev-parse', 'HEAD'], show_stdout=False, cwd=location, + ['rev-parse', rev], show_stdout=False, cwd=location, ) return current_rev.strip() @@ -255,17 +294,15 @@ class Git(VersionControl): repo = self.get_url(location) if not repo.lower().startswith('git:'): repo = 'git+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None current_rev = self.get_revision(location) - req = '%s@%s#egg=%s' % (repo, current_rev, egg_project_name) - subdirectory = self._get_subdirectory(location) - if subdirectory: - req += '&subdirectory=' + subdirectory + egg_project_name = dist.egg_name().split('-', 1)[0] + subdir = self._get_subdirectory(location) + req = make_vcs_requirement_url(repo, current_rev, egg_project_name, + subdir=subdir) + return req - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): """ Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. That's required because although they use SSH they sometimes don't @@ -275,12 +312,12 @@ class Git(VersionControl): if '://' not in url: assert 'file:' not in url url = url.replace('git+', 'git+ssh://') - url, rev = super(Git, self).get_url_rev(url) + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) url = url.replace('ssh://', '') else: - url, rev = super(Git, self).get_url_rev(url) + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) - return url, rev + return url, rev, user_pass def update_submodules(self, location): if not os.path.exists(os.path.join(location, '.gitmodules')): diff --git a/pipenv/patched/notpip/_internal/vcs/mercurial.py b/pipenv/patched/notpip/_internal/vcs/mercurial.py index a143e765..d76d47f3 100644 --- a/pipenv/patched/notpip/_internal/vcs/mercurial.py +++ b/pipenv/patched/notpip/_internal/vcs/mercurial.py @@ -6,7 +6,7 @@ import os from pipenv.patched.notpip._vendor.six.moves import configparser from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -59,7 +59,7 @@ class Mercurial(VersionControl): cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): self.run_command(['pull', '-q'], cwd=dest) cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) @@ -88,11 +88,10 @@ class Mercurial(VersionControl): repo = self.get_url(location) if not repo.lower().startswith('hg:'): repo = 'hg+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None current_rev_hash = self.get_revision_hash(location) - return '%s@%s#egg=%s' % (repo, current_rev_hash, egg_project_name) + egg_project_name = dist.egg_name().split('-', 1)[0] + return make_vcs_requirement_url(repo, current_rev_hash, + egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/subversion.py b/pipenv/patched/notpip/_internal/vcs/subversion.py index 5adbdaa3..f3c3db4d 100644 --- a/pipenv/patched/notpip/_internal/vcs/subversion.py +++ b/pipenv/patched/notpip/_internal/vcs/subversion.py @@ -4,17 +4,15 @@ import logging import os import re -from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse - -from pipenv.patched.notpip._internal.index import Link +from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.logging import indent_log -from pipenv.patched.notpip._internal.utils.misc import display_path, remove_auth_from_url, rmtree +from pipenv.patched.notpip._internal.utils.misc import ( + display_path, make_vcs_requirement_url, rmtree, split_auth_from_netloc, +) from pipenv.patched.notpip._internal.vcs import VersionControl, vcs _svn_xml_url_re = re.compile('url="([^"]+)"') _svn_rev_re = re.compile(r'committed-rev="(\d+)"') -_svn_url_re = re.compile(r'URL: (.+)') -_svn_revision_re = re.compile(r'Revision: (.+)') _svn_info_xml_rev_re = re.compile(r'\s*revision="(\d+)"') _svn_info_xml_url_re = re.compile(r'(.*)') @@ -31,34 +29,6 @@ class Subversion(VersionControl): def get_base_rev_args(self, rev): return ['-r', rev] - def get_info(self, location): - """Returns (url, revision), where both are strings""" - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - output = self.run_command( - ['info', location], - show_stdout=False, - extra_environ={'LANG': 'C'}, - ) - match = _svn_url_re.search(output) - if not match: - logger.warning( - 'Cannot determine URL of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return None, None - url = match.group(1).strip() - match = _svn_revision_re.search(output) - if not match: - logger.warning( - 'Cannot determine revision of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return url, None - return url, match.group(1) - def export(self, location): """Export the svn repository at the url to the destination location""" url, rev_options = self.get_url_rev_options(self.url) @@ -87,7 +57,7 @@ class Subversion(VersionControl): cmd_args = ['switch'] + rev_options.to_args() + [url, dest] self.run_command(cmd_args) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): cmd_args = ['update'] + rev_options.to_args() + [dest] self.run_command(cmd_args) @@ -132,18 +102,34 @@ class Subversion(VersionControl): revision = max(revision, localrev) return revision - def get_url_rev(self, url): + def get_netloc_and_auth(self, netloc, scheme): + """ + This override allows the auth information to be passed to svn via the + --username and --password options instead of via the URL. + """ + if scheme == 'ssh': + # The --username and --password options can't be used for + # svn+ssh URLs, so keep the auth information in the URL. + return super(Subversion, self).get_netloc_and_auth( + netloc, scheme) + + return split_auth_from_netloc(netloc) + + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it - url, rev = super(Subversion, self).get_url_rev(url) + url, rev, user_pass = super(Subversion, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'svn+' + url - return url, rev + return url, rev, user_pass - def get_url_rev_args(self, url): - extra_args = get_rev_options_args(url) - url = remove_auth_from_url(url) + def make_rev_args(self, username, password): + extra_args = [] + if username: + extra_args += ['--username', username] + if password: + extra_args += ['--password', password] - return url, extra_args + return extra_args def get_url(self, location): # In cases where the source is in a subdirectory, not alongside @@ -213,42 +199,15 @@ class Subversion(VersionControl): repo = self.get_url(location) if repo is None: return None + repo = 'svn+' + repo + rev = self.get_revision(location) # FIXME: why not project name? egg_project_name = dist.egg_name().split('-', 1)[0] - rev = self.get_revision(location) - return 'svn+%s@%s#egg=%s' % (repo, rev, egg_project_name) + return make_vcs_requirement_url(repo, rev, egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" return False -def get_rev_options_args(url): - """ - Return the extra arguments to pass to RevOptions. - """ - r = urllib_parse.urlsplit(url) - if hasattr(r, 'username'): - # >= Python-2.5 - username, password = r.username, r.password - else: - netloc = r[1] - if '@' in netloc: - auth = netloc.split('@')[0] - if ':' in auth: - username, password = auth.split(':', 1) - else: - username, password = auth, None - else: - username, password = None, None - - extra_args = [] - if username: - extra_args += ['--username', username] - if password: - extra_args += ['--password', password] - - return extra_args - - vcs.register(Subversion) diff --git a/pipenv/patched/notpip/_internal/wheel.py b/pipenv/patched/notpip/_internal/wheel.py index 14ec0014..6df5a3a3 100644 --- a/pipenv/patched/notpip/_internal/wheel.py +++ b/pipenv/patched/notpip/_internal/wheel.py @@ -167,7 +167,8 @@ def message_about_scripts_not_on_PATH(scripts): ] # If an executable sits with sys.executable, we don't warn for it. # This covers the case of venv invocations without activating the venv. - not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) + executable_loc = os.environ.get("PIP_PYTHON_PATH", sys.executable) + not_warn_dirs.append(os.path.normcase(os.path.dirname(executable_loc))) warn_for = { parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() if os.path.normcase(parent_dir) not in not_warn_dirs @@ -475,7 +476,7 @@ if __name__ == '__main__': if warn_script_location: msg = message_about_scripts_not_on_PATH(generated_console_scripts) if msg is not None: - logger.warn(msg) + logger.warning(msg) if len(gui) > 0: generated.extend( @@ -500,16 +501,19 @@ if __name__ == '__main__': with open_for_csv(temp_record, 'w+') as record_out: reader = csv.reader(record_in) writer = csv.writer(record_out) + outrows = [] for row in reader: row[0] = installed.pop(row[0], row[0]) if row[0] in changed: row[1], row[2] = rehash(row[0]) - writer.writerow(row) + outrows.append(tuple(row)) for f in generated: digest, length = rehash(f) - writer.writerow((normpath(f, lib_dir), digest, length)) + outrows.append((normpath(f, lib_dir), digest, length)) for f in installed: - writer.writerow((installed[f], '', '')) + outrows.append((installed[f], '', '')) + for row in sorted(outrows): + writer.writerow(row) shutil.move(temp_record, record) @@ -664,8 +668,9 @@ class WheelBuilder(object): # isolating. Currently, it breaks Python in virtualenvs, because it # relies on site.py to find parts of the standard library outside the # virtualenv. + executable_loc = os.environ.get('PIP_PYTHON_PATH', sys.executable) return [ - os.environ.get('PIP_PYTHON_PATH', sys.executable), '-u', '-c', + executable_loc, '-u', '-c', SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) @@ -710,6 +715,7 @@ class WheelBuilder(object): :return: True if all the wheels built correctly. """ from pipenv.patched.notpip._internal import index + from pipenv.patched.notpip._internal.models.link import Link building_is_possible = self._wheel_dir or ( autobuilding and self.wheel_cache.cache_dir @@ -717,6 +723,7 @@ class WheelBuilder(object): assert building_is_possible buildset = [] + format_control = self.finder.format_control for req in requirements: if req.constraint: continue @@ -740,8 +747,7 @@ class WheelBuilder(object): if index.egg_info_matches(base, None, link) is None: # E.g. local directory. Build wheel just for this run. ephem_cache = True - if "binary" not in index.fmt_ctl_formats( - self.finder.format_control, + if "binary" not in format_control.get_allowed_formats( canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " @@ -802,7 +808,7 @@ class WheelBuilder(object): self.preparer.build_dir ) # Update the link for this. - req.link = index.Link(path_to_url(wheel_file)) + req.link = Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( diff --git a/pipenv/patched/notpip/_vendor/certifi/__init__.py b/pipenv/patched/notpip/_vendor/certifi/__init__.py index 0c4963ef..aa329fbb 100644 --- a/pipenv/patched/notpip/_vendor/certifi/__init__.py +++ b/pipenv/patched/notpip/_vendor/certifi/__init__.py @@ -1,3 +1,3 @@ from .core import where, old_where -__version__ = "2018.04.16" +__version__ = "2018.08.24" diff --git a/pipenv/patched/notpip/_vendor/certifi/__main__.py b/pipenv/patched/notpip/_vendor/certifi/__main__.py index 5f1da0dd..983ed0f9 100644 --- a/pipenv/patched/notpip/_vendor/certifi/__main__.py +++ b/pipenv/patched/notpip/_vendor/certifi/__main__.py @@ -1,2 +1,2 @@ -from certifi import where +from pipenv.patched.notpip._vendor.certifi import where print(where()) diff --git a/pipenv/patched/notpip/_vendor/certifi/cacert.pem b/pipenv/patched/notpip/_vendor/certifi/cacert.pem index 2713f541..85de024e 100644 --- a/pipenv/patched/notpip/_vendor/certifi/cacert.pem +++ b/pipenv/patched/notpip/_vendor/certifi/cacert.pem @@ -3692,169 +3692,6 @@ lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR -----END CERTIFICATE----- -# Issuer: CN=Certplus Root CA G1 O=Certplus -# Subject: CN=Certplus Root CA G1 O=Certplus -# Label: "Certplus Root CA G1" -# Serial: 1491911565779898356709731176965615564637713 -# MD5 Fingerprint: 7f:09:9c:f7:d9:b9:5c:69:69:56:d5:37:3e:14:0d:42 -# SHA1 Fingerprint: 22:fd:d0:b7:fd:a2:4e:0d:ac:49:2c:a0:ac:a6:7b:6a:1f:e3:f7:66 -# SHA256 Fingerprint: 15:2a:40:2b:fc:df:2c:d5:48:05:4d:22:75:b3:9c:7f:ca:3e:c0:97:80:78:b0:f0:ea:76:e5:61:a6:c7:43:3e ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgISESBVg+QtPlRWhS2DN7cs3EYRMA0GCSqGSIb3DQEBDQUA -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBa -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB -ANpQh7bauKk+nWT6VjOaVj0W5QOVsjQcmm1iBdTYj+eJZJ+622SLZOZ5KmHNr49a -iZFluVj8tANfkT8tEBXgfs+8/H9DZ6itXjYj2JizTfNDnjl8KvzsiNWI7nC9hRYt -6kuJPKNxQv4c/dMcLRC4hlTqQ7jbxofaqK6AJc96Jh2qkbBIb6613p7Y1/oA/caP -0FG7Yn2ksYyy/yARujVjBYZHYEMzkPZHogNPlk2dT8Hq6pyi/jQu3rfKG3akt62f -6ajUeD94/vI4CTYd0hYCyOwqaK/1jpTvLRN6HkJKHRUxrgwEV/xhc/MxVoYxgKDE -EW4wduOU8F8ExKyHcomYxZ3MVwia9Az8fXoFOvpHgDm2z4QTd28n6v+WZxcIbekN -1iNQMLAVdBM+5S//Ds3EC0pd8NgAM0lm66EYfFkuPSi5YXHLtaW6uOrc4nBvCGrc -h2c0798wct3zyT8j/zXhviEpIDCB5BmlIOklynMxdCm+4kLV87ImZsdo/Rmz5yCT -mehd4F6H50boJZwKKSTUzViGUkAksnsPmBIgJPaQbEfIDbsYIC7Z/fyL8inqh3SV -4EJQeIQEQWGw9CEjjy3LKCHyamz0GqbFFLQ3ZU+V/YDI+HLlJWvEYLF7bY5KinPO -WftwenMGE9nTdDckQQoRb5fc5+R+ob0V8rqHDz1oihYHAgMBAAGjYzBhMA4GA1Ud -DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSowcCbkahDFXxd -Bie0KlHYlwuBsTAfBgNVHSMEGDAWgBSowcCbkahDFXxdBie0KlHYlwuBsTANBgkq -hkiG9w0BAQ0FAAOCAgEAnFZvAX7RvUz1isbwJh/k4DgYzDLDKTudQSk0YcbX8ACh -66Ryj5QXvBMsdbRX7gp8CXrc1cqh0DQT+Hern+X+2B50ioUHj3/MeXrKls3N/U/7 -/SMNkPX0XtPGYX2eEeAC7gkE2Qfdpoq3DIMku4NQkv5gdRE+2J2winq14J2by5BS -S7CTKtQ+FjPlnsZlFT5kOwQ/2wyPX1wdaR+v8+khjPPvl/aatxm2hHSco1S1cE5j -2FddUyGbQJJD+tZ3VTNPZNX70Cxqjm0lpu+F6ALEUz65noe8zDUa3qHpimOHZR4R -Kttjd5cUvpoUmRGywO6wT/gUITJDT5+rosuoD6o7BlXGEilXCNQ314cnrUlZp5Gr -RHpejXDbl85IULFzk/bwg2D5zfHhMf1bfHEhYxQUqq/F3pN+aLHsIqKqkHWetUNy -6mSjhEv9DKgma3GX7lZjZuhCVPnHHd/Qj1vfyDBviP4NxDMcU6ij/UgQ8uQKTuEV -V/xuZDDCVRHc6qnNSlSsKWNEz0pAoNZoWRsz+e86i9sgktxChL8Bq4fA1SCC28a5 -g4VCXA9DO2pJNdWY9BW/+mGBDAkgGNLQFwzLSABQ6XaCjGTXOqAHVcweMcDvOrRl -++O/QmueD6i9a5jc2NvLi6Td11n0bt3+qsOR0C5CB8AMTVPNJLFMWx5R9N/pkvo= ------END CERTIFICATE----- - -# Issuer: CN=Certplus Root CA G2 O=Certplus -# Subject: CN=Certplus Root CA G2 O=Certplus -# Label: "Certplus Root CA G2" -# Serial: 1492087096131536844209563509228951875861589 -# MD5 Fingerprint: a7:ee:c4:78:2d:1b:ee:2d:b9:29:ce:d6:a7:96:32:31 -# SHA1 Fingerprint: 4f:65:8e:1f:e9:06:d8:28:02:e9:54:47:41:c9:54:25:5d:69:cc:1a -# SHA256 Fingerprint: 6c:c0:50:41:e6:44:5e:74:69:6c:4c:fb:c9:f8:0f:54:3b:7e:ab:bb:44:b4:ce:6f:78:7c:6a:99:71:c4:2f:17 ------BEGIN CERTIFICATE----- -MIICHDCCAaKgAwIBAgISESDZkc6uo+jF5//pAq/Pc7xVMAoGCCqGSM49BAMDMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjB2MBAGByqGSM49AgEGBSuBBAAiA2IABM0PW1aC3/BFGtat -93nwHcmsltaeTpwftEIRyoa/bfuFo8XlGVzX7qY/aWfYeOKmycTbLXku54uNAm8x -Ik0G42ByRZ0OQneezs/lf4WbGOT8zC5y0xaTTsqZY1yhBSpsBqNjMGEwDgYDVR0P -AQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNqDYwJ5jtpMxjwj -FNiPwyCrKGBZMB8GA1UdIwQYMBaAFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMAoGCCqG -SM49BAMDA2gAMGUCMHD+sAvZ94OX7PNVHdTcswYO/jOYnYs5kGuUIe22113WTNch -p+e/IQ8rzfcq3IUHnQIxAIYUFuXcsGXCwI4Un78kFmjlvPl5adytRSv3tjFzzAal -U5ORGpOucGpnutee5WEaXw== ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G1 O=OpenTrust -# Subject: CN=OpenTrust Root CA G1 O=OpenTrust -# Label: "OpenTrust Root CA G1" -# Serial: 1492036577811947013770400127034825178844775 -# MD5 Fingerprint: 76:00:cc:81:29:cd:55:5e:88:6a:7a:2e:f7:4d:39:da -# SHA1 Fingerprint: 79:91:e8:34:f7:e2:ee:dd:08:95:01:52:e9:55:2d:14:e9:58:d5:7e -# SHA256 Fingerprint: 56:c7:71:28:d9:8c:18:d9:1b:4c:fd:ff:bc:25:ee:91:03:d4:75:8e:a2:ab:ad:82:6a:90:f3:45:7d:46:0e:b4 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESCzkFU5fX82bWTCp59rY45nMA0GCSqGSIb3DQEBCwUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcxMB4XDTE0MDUyNjA4NDU1MFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQD4eUbalsUwXopxAy1wpLuwxQjczeY1wICkES3d5oeuXT2R0odsN7faYp6b -wiTXj/HbpqbfRm9RpnHLPhsxZ2L3EVs0J9V5ToybWL0iEA1cJwzdMOWo010hOHQX -/uMftk87ay3bfWAfjH1MBcLrARYVmBSO0ZB3Ij/swjm4eTrwSSTilZHcYTSSjFR0 -77F9jAHiOH3BX2pfJLKOYheteSCtqx234LSWSE9mQxAGFiQD4eCcjsZGT44ameGP -uY4zbGneWK2gDqdkVBFpRGZPTBKnjix9xNRbxQA0MMHZmf4yzgeEtE7NCv82TWLx -p2NX5Ntqp66/K7nJ5rInieV+mhxNaMbBGN4zK1FGSxyO9z0M+Yo0FMT7MzUj8czx -Kselu7Cizv5Ta01BG2Yospb6p64KTrk5M0ScdMGTHPjgniQlQ/GbI4Kq3ywgsNw2 -TgOzfALU5nsaqocTvz6hdLubDuHAk5/XpGbKuxs74zD0M1mKB3IDVedzagMxbm+W -G+Oin6+Sx+31QrclTDsTBM8clq8cIqPQqwWyTBIjUtz9GVsnnB47ev1CI9sjgBPw -vFEVVJSmdz7QdFG9URQIOTfLHzSpMJ1ShC5VkLG631UAC9hWLbFJSXKAqWLXwPYY -EQRVzXR7z2FwefR7LFxckvzluFqrTJOVoSfupb7PcSNCupt2LQIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUl0YhVyE1 -2jZVx/PxN3DlCPaTKbYwHwYDVR0jBBgwFoAUl0YhVyE12jZVx/PxN3DlCPaTKbYw -DQYJKoZIhvcNAQELBQADggIBAB3dAmB84DWn5ph76kTOZ0BP8pNuZtQ5iSas000E -PLuHIT839HEl2ku6q5aCgZG27dmxpGWX4m9kWaSW7mDKHyP7Rbr/jyTwyqkxf3kf -gLMtMrpkZ2CvuVnN35pJ06iCsfmYlIrM4LvgBBuZYLFGZdwIorJGnkSI6pN+VxbS -FXJfLkur1J1juONI5f6ELlgKn0Md/rcYkoZDSw6cMoYsYPXpSOqV7XAp8dUv/TW0 -V8/bhUiZucJvbI/NeJWsZCj9VrDDb8O+WVLhX4SPgPL0DTatdrOjteFkdjpY3H1P -XlZs5VVZV6Xf8YpmMIzUUmI4d7S+KNfKNsSbBfD4Fdvb8e80nR14SohWZ25g/4/I -i+GOvUKpMwpZQhISKvqxnUOOBZuZ2mKtVzazHbYNeS2WuOvyDEsMpZTGMKcmGS3t -TAZQMPH9WD25SxdfGbRqhFS0OE85og2WaMMolP3tLR9Ka0OWLpABEPs4poEL0L91 -09S5zvE/bw4cHjdx5RiHdRk/ULlepEU0rbDK5uUTdg8xFKmOLZTW1YVNcxVPS/Ky -Pu1svf0OnWZzsD2097+o4BGkxK51CUpjAEggpsadCwmKtODmzj7HPiY46SvepghJ -AwSQiumPv+i2tCqjI40cHLI5kqiPAlxAOXXUc0ECd97N4EOH1uS6SsNsEn/+KuYj -1oxx ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G2 O=OpenTrust -# Subject: CN=OpenTrust Root CA G2 O=OpenTrust -# Label: "OpenTrust Root CA G2" -# Serial: 1492012448042702096986875987676935573415441 -# MD5 Fingerprint: 57:24:b6:59:24:6b:ae:c8:fe:1c:0c:20:f2:c0:4e:eb -# SHA1 Fingerprint: 79:5f:88:60:c5:ab:7c:3d:92:e6:cb:f4:8d:e1:45:cd:11:ef:60:0b -# SHA256 Fingerprint: 27:99:58:29:fe:6a:75:15:c1:bf:e8:48:f9:c4:76:1d:b1:6c:22:59:29:25:7b:f4:0d:08:94:f2:9e:a8:ba:f2 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESChaRu/vbm9UpaPI+hIvyYRMA0GCSqGSIb3DQEBDQUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcyMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQDMtlelM5QQgTJT32F+D3Y5z1zCU3UdSXqWON2ic2rxb95eolq5cSG+Ntmh -/LzubKh8NBpxGuga2F8ORAbtp+Dz0mEL4DKiltE48MLaARf85KxP6O6JHnSrT78e -CbY2albz4e6WiWYkBuTNQjpK3eCasMSCRbP+yatcfD7J6xcvDH1urqWPyKwlCm/6 -1UWY0jUJ9gNDlP7ZvyCVeYCYitmJNbtRG6Q3ffyZO6v/v6wNj0OxmXsWEH4db0fE -FY8ElggGQgT4hNYdvJGmQr5J1WqIP7wtUdGejeBSzFfdNTVY27SPJIjki9/ca1TS -gSuyzpJLHB9G+h3Ykst2Z7UJmQnlrBcUVXDGPKBWCgOz3GIZ38i1MH/1PCZ1Eb3X -G7OHngevZXHloM8apwkQHZOJZlvoPGIytbU6bumFAYueQ4xncyhZW+vj3CzMpSZy -YhK05pyDRPZRpOLAeiRXyg6lPzq1O4vldu5w5pLeFlwoW5cZJ5L+epJUzpM5ChaH -vGOz9bGTXOBut9Dq+WIyiET7vycotjCVXRIouZW+j1MY5aIYFuJWpLIsEPUdN6b4 -t/bQWVyJ98LVtZR00dX+G7bw5tYee9I8y6jj9RjzIR9u701oBnstXW5DiabA+aC/ -gh7PU3+06yzbXfZqfUAkBXKJOAGTy3HCOV0GEfZvePg3DTmEJwIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUajn6QiL3 -5okATV59M4PLuG53hq8wHwYDVR0jBBgwFoAUajn6QiL35okATV59M4PLuG53hq8w -DQYJKoZIhvcNAQENBQADggIBAJjLq0A85TMCl38th6aP1F5Kr7ge57tx+4BkJamz -Gj5oXScmp7oq4fBXgwpkTx4idBvpkF/wrM//T2h6OKQQbA2xx6R3gBi2oihEdqc0 -nXGEL8pZ0keImUEiyTCYYW49qKgFbdEfwFFEVn8nNQLdXpgKQuswv42hm1GqO+qT -RmTFAHneIWv2V6CG1wZy7HBGS4tz3aAhdT7cHcCP009zHIXZ/n9iyJVvttN7jLpT -wm+bREx50B1ws9efAvSyB7DH5fitIw6mVskpEndI2S9G/Tvw/HRwkqWOOAgfZDC2 -t0v7NqwQjqBSM2OdAzVWxWm9xiNaJ5T2pBL4LTM8oValX9YZ6e18CL13zSdkzJTa -TkZQh+D5wVOAHrut+0dSixv9ovneDiK3PTNZbNTe9ZUGMg1RGUFcPk8G97krgCf2 -o6p6fAbhQ8MTOWIaNr3gKC6UAuQpLmBVrkA9sHSSXvAgZJY/X0VdiLWK2gKgW0VU -3jg9CcCoSmVGFvyqv1ROTVu+OEO3KMqLM6oaJbolXCkvW0pujOotnCr2BXbgd5eA -iN1nE28daCSLT7d0geX0YJ96Vdc+N9oWaz53rK4YcJUIeSkDiv7BO7M/Gg+kO14f -WKGVyasvc0rQLW6aWQ9VGHgtPFGml4vmu7JwqkwR3v98KzfUetF3NI/n+UL3PIEM -S1IK ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G3 O=OpenTrust -# Subject: CN=OpenTrust Root CA G3 O=OpenTrust -# Label: "OpenTrust Root CA G3" -# Serial: 1492104908271485653071219941864171170455615 -# MD5 Fingerprint: 21:37:b4:17:16:92:7b:67:46:70:a9:96:d7:a8:13:24 -# SHA1 Fingerprint: 6e:26:64:f3:56:bf:34:55:bf:d1:93:3f:7c:01:de:d8:13:da:8a:a6 -# SHA256 Fingerprint: b7:c3:62:31:70:6e:81:07:8c:36:7c:b8:96:19:8f:1e:32:08:dd:92:69:49:dd:8f:57:09:a4:10:f7:5b:62:92 ------BEGIN CERTIFICATE----- -MIICITCCAaagAwIBAgISESDm+Ez8JLC+BUCs2oMbNGA/MAoGCCqGSM49BAMDMEAx -CzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5U -cnVzdCBSb290IENBIEczMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFow -QDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwUT3Bl -blRydXN0IFJvb3QgQ0EgRzMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARK7liuTcpm -3gY6oxH84Bjwbhy6LTAMidnW7ptzg6kjFYwvWYpa3RTqnVkrQ7cG7DK2uu5Bta1d -oYXM6h0UZqNnfkbilPPntlahFVmhTzeXuSIevRHr9LIfXsMUmuXZl5mjYzBhMA4G -A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRHd8MUi2I5 -DMlv4VBN0BBY3JWIbTAfBgNVHSMEGDAWgBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAK -BggqhkjOPQQDAwNpADBmAjEAj6jcnboMBBf6Fek9LykBl7+BFjNAk2z8+e2AcG+q -j9uEwov1NcoG3GRvaBbhj5G5AjEA2Euly8LQCGzpGPta3U1fJAuwACEl74+nBCZx -4nxp5V2a+EEfOzmTk51V6s2N8fvB ------END CERTIFICATE----- - # Issuer: CN=ISRG Root X1 O=Internet Security Research Group # Subject: CN=ISRG Root X1 O=Internet Security Research Group # Label: "ISRG Root X1" @@ -4398,3 +4235,66 @@ MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg== -----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Label: "GlobalSign Root CA - R6" +# Serial: 1417766617973444989252670301619537 +# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae +# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1 +# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69 +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg +MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx +MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET +MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI +xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k +ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD +aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw +LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw +1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX +k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 +SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h +bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n +WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY +rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce +MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu +bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN +nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt +Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 +55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj +vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf +cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz +oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp +nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs +pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v +JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R +8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 +5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= +-----END CERTIFICATE----- + +# Issuer: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Subject: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Label: "OISTE WISeKey Global Root GC CA" +# Serial: 44084345621038548146064804565436152554 +# MD5 Fingerprint: a9:d6:b9:2d:2f:93:64:f8:a5:69:ca:91:e9:68:07:23 +# SHA1 Fingerprint: e0:11:84:5e:34:de:be:88:81:b9:9c:f6:16:26:d1:96:1f:c3:b9:31 +# SHA256 Fingerprint: 85:60:f9:1c:36:24:da:ba:95:70:b5:fe:a0:db:e3:6f:f1:1a:83:23:be:94:86:85:4f:b3:f3:4a:55:71:19:8d +-----BEGIN CERTIFICATE----- +MIICaTCCAe+gAwIBAgIQISpWDK7aDKtARb8roi066jAKBggqhkjOPQQDAzBtMQsw +CQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUgRm91 +bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9iYWwg +Um9vdCBHQyBDQTAeFw0xNzA1MDkwOTQ4MzRaFw00MjA1MDkwOTU4MzNaMG0xCzAJ +BgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBGb3Vu +ZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2JhbCBS +b290IEdDIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAETOlQwMYPchi82PG6s4ni +eUqjFqdrVCTbUf/q9Akkwwsin8tqJ4KBDdLArzHkdIJuyiXZjHWd8dvQmqJLIX4W +p2OQ0jnUsYd4XxiWD1AbNTcPasbc2RNNpI6QN+a9WzGRo1QwUjAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUSIcUrOPDnpBgOtfKie7T +rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV +57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg +Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 +-----END CERTIFICATE----- diff --git a/pipenv/patched/notpip/_vendor/packaging/__about__.py b/pipenv/patched/notpip/_vendor/packaging/__about__.py index 4255c5b5..21fc6ce3 100644 --- a/pipenv/patched/notpip/_vendor/packaging/__about__.py +++ b/pipenv/patched/notpip/_vendor/packaging/__about__.py @@ -12,10 +12,10 @@ __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "17.1" +__version__ = "18.0" __author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" __license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2014-2016 %s" % __author__ +__copyright__ = "Copyright 2014-2018 %s" % __author__ diff --git a/pipenv/patched/notpip/_vendor/packaging/requirements.py b/pipenv/patched/notpip/_vendor/packaging/requirements.py index 0da25914..5ec5d74a 100644 --- a/pipenv/patched/notpip/_vendor/packaging/requirements.py +++ b/pipenv/patched/notpip/_vendor/packaging/requirements.py @@ -92,16 +92,16 @@ class Requirement(object): try: req = REQUIREMENT.parseString(requirement_string) except ParseException as e: - raise InvalidRequirement( - "Invalid requirement, parse error at \"{0!r}\"".format( - requirement_string[e.loc:e.loc + 8])) + raise InvalidRequirement("Parse error at \"{0!r}\": {1}".format( + requirement_string[e.loc:e.loc + 8], e.msg + )) self.name = req.name if req.url: parsed_url = urlparse.urlparse(req.url) if not (parsed_url.scheme and parsed_url.netloc) or ( not parsed_url.scheme and not parsed_url.netloc): - raise InvalidRequirement("Invalid URL given") + raise InvalidRequirement("Invalid URL: {0}".format(req.url)) self.url = req.url else: self.url = None diff --git a/pipenv/patched/notpip/_vendor/packaging/specifiers.py b/pipenv/patched/notpip/_vendor/packaging/specifiers.py index 9b6353f0..4c798999 100644 --- a/pipenv/patched/notpip/_vendor/packaging/specifiers.py +++ b/pipenv/patched/notpip/_vendor/packaging/specifiers.py @@ -503,7 +503,7 @@ class Specifier(_IndividualSpecifier): return False # Ensure that we do not allow a local version of the version mentioned - # in the specifier, which is techincally greater than, to match. + # in the specifier, which is technically greater than, to match. if prospective.local is not None: if Version(prospective.base_version) == Version(spec.base_version): return False diff --git a/pipenv/vendor/pathlib2.LICENSE.rst b/pipenv/patched/notpip/_vendor/pep517/LICENSE similarity index 83% rename from pipenv/vendor/pathlib2.LICENSE.rst rename to pipenv/patched/notpip/_vendor/pep517/LICENSE index ddb51b8a..b0ae9dbc 100644 --- a/pipenv/vendor/pathlib2.LICENSE.rst +++ b/pipenv/patched/notpip/_vendor/pep517/LICENSE @@ -1,7 +1,6 @@ The MIT License (MIT) -Copyright (c) 2014 Matthias C. M. Troffaes -Copyright (c) 2012-2014 Antoine Pitrou and contributors +Copyright (c) 2017 Thomas Kluyver Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -10,14 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/patched/notpip/_vendor/pep517/__init__.py b/pipenv/patched/notpip/_vendor/pep517/__init__.py new file mode 100644 index 00000000..8beedea4 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/__init__.py @@ -0,0 +1,4 @@ +"""Wrappers to build Python packages using PEP 517 hooks +""" + +__version__ = '0.2' diff --git a/pipenv/patched/notpip/_vendor/pep517/_in_process.py b/pipenv/patched/notpip/_vendor/pep517/_in_process.py new file mode 100644 index 00000000..baa14d38 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/_in_process.py @@ -0,0 +1,182 @@ +"""This is invoked in a subprocess to call the build backend hooks. + +It expects: +- Command line args: hook_name, control_dir +- Environment variable: PEP517_BUILD_BACKEND=entry.point:spec +- control_dir/input.json: + - {"kwargs": {...}} + +Results: +- control_dir/output.json + - {"return_val": ...} +""" +from glob import glob +from importlib import import_module +import os +from os.path import join as pjoin +import re +import shutil +import sys + +# This is run as a script, not a module, so it can't do a relative import +import compat + +def _build_backend(): + """Find and load the build backend""" + ep = os.environ['PEP517_BUILD_BACKEND'] + mod_path, _, obj_path = ep.partition(':') + obj = import_module(mod_path) + if obj_path: + for path_part in obj_path.split('.'): + obj = getattr(obj, path_part) + return obj + +def get_requires_for_build_wheel(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_wheel + except AttributeError: + return [] + else: + return hook(config_settings) + +def prepare_metadata_for_build_wheel(metadata_directory, config_settings): + """Invoke optional prepare_metadata_for_build_wheel + + Implements a fallback by building a wheel if the hook isn't defined. + """ + backend = _build_backend() + try: + hook = backend.prepare_metadata_for_build_wheel + except AttributeError: + return _get_wheel_metadata_from_wheel(backend, metadata_directory, + config_settings) + else: + return hook(metadata_directory, config_settings) + +WHEEL_BUILT_MARKER = 'PEP517_ALREADY_BUILT_WHEEL' + +def _dist_info_files(whl_zip): + """Identify the .dist-info folder inside a wheel ZipFile.""" + res = [] + for path in whl_zip.namelist(): + m = re.match(r'[^/\\]+-[^/\\]+\.dist-info/', path) + if m: + res.append(path) + if res: + return res + raise Exception("No .dist-info folder found in wheel") + +def _get_wheel_metadata_from_wheel(backend, metadata_directory, config_settings): + """Build a wheel and extract the metadata from it. + + Fallback for when the build backend does not define the 'get_wheel_metadata' + hook. + """ + from zipfile import ZipFile + whl_basename = backend.build_wheel(metadata_directory, config_settings) + with open(os.path.join(metadata_directory, WHEEL_BUILT_MARKER), 'wb'): + pass # Touch marker file + + whl_file = os.path.join(metadata_directory, whl_basename) + with ZipFile(whl_file) as zipf: + dist_info = _dist_info_files(zipf) + zipf.extractall(path=metadata_directory, members=dist_info) + return dist_info[0].split('/')[0] + +def _find_already_built_wheel(metadata_directory): + """Check for a wheel already built during the get_wheel_metadata hook. + """ + if not metadata_directory: + return None + metadata_parent = os.path.dirname(metadata_directory) + if not os.path.isfile(pjoin(metadata_parent, WHEEL_BUILT_MARKER)): + return None + + whl_files = glob(os.path.join(metadata_parent, '*.whl')) + if not whl_files: + print('Found wheel built marker, but no .whl files') + return None + if len(whl_files) > 1: + print('Found multiple .whl files; unspecified behaviour. ' + 'Will call build_wheel.') + return None + + # Exactly one .whl file + return whl_files[0] + +def build_wheel(wheel_directory, config_settings, metadata_directory=None): + """Invoke the mandatory build_wheel hook. + + If a wheel was already built in the prepare_metadata_for_build_wheel fallback, this + will copy it rather than rebuilding the wheel. + """ + prebuilt_whl = _find_already_built_wheel(metadata_directory) + if prebuilt_whl: + shutil.copy2(prebuilt_whl, wheel_directory) + return os.path.basename(prebuilt_whl) + + return _build_backend().build_wheel(wheel_directory, config_settings, + metadata_directory) + + +def get_requires_for_build_sdist(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_sdist + except AttributeError: + return [] + else: + return hook(config_settings) + +class _DummyException(Exception): + """Nothing should ever raise this exception""" + +class GotUnsupportedOperation(Exception): + """For internal use when backend raises UnsupportedOperation""" + +def build_sdist(sdist_directory, config_settings): + """Invoke the mandatory build_sdist hook.""" + backend = _build_backend() + try: + return backend.build_sdist(sdist_directory, config_settings) + except getattr(backend, 'UnsupportedOperation', _DummyException): + raise GotUnsupportedOperation + +HOOK_NAMES = { + 'get_requires_for_build_wheel', + 'prepare_metadata_for_build_wheel', + 'build_wheel', + 'get_requires_for_build_sdist', + 'build_sdist', +} + +def main(): + if len(sys.argv) < 3: + sys.exit("Needs args: hook_name, control_dir") + hook_name = sys.argv[1] + control_dir = sys.argv[2] + if hook_name not in HOOK_NAMES: + sys.exit("Unknown hook: %s" % hook_name) + hook = globals()[hook_name] + + hook_input = compat.read_json(pjoin(control_dir, 'input.json')) + + json_out = {'unsupported': False, 'return_val': None} + try: + json_out['return_val'] = hook(**hook_input['kwargs']) + except GotUnsupportedOperation: + json_out['unsupported'] = True + + compat.write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) + +if __name__ == '__main__': + main() diff --git a/pipenv/patched/notpip/_vendor/pep517/check.py b/pipenv/patched/notpip/_vendor/pep517/check.py new file mode 100644 index 00000000..3dffd2e0 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/check.py @@ -0,0 +1,194 @@ +"""Check a project and backend by attempting to build using PEP 517 hooks. +""" +import argparse +import logging +import os +from os.path import isfile, join as pjoin +from pipenv.patched.notpip._vendor.pytoml import TomlError, load as toml_load +import shutil +from subprocess import CalledProcessError +import sys +import tarfile +from tempfile import mkdtemp +import zipfile + +from .colorlog import enable_colourful_output +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + +def check_build_sdist(hooks): + with BuildEnvironment() as env: + try: + env.pip_install(hooks.build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_sdist({}) + log.info('Got build requires: %s', reqs) + except: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build sdist in %s', td) + try: + try: + filename = hooks.build_sdist(td, {}) + log.info('build_sdist returned %r', filename) + except: + log.info('Failure in build_sdist', exc_info=True) + return False + + if not filename.endswith('.tar.gz'): + log.error("Filename %s doesn't have .tar.gz extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if tarfile.is_tarfile(path): + log.info("Output file is a tar file") + else: + log.error("Output file is not a tar file") + return False + + finally: + shutil.rmtree(td) + + return True + +def check_build_wheel(hooks): + with BuildEnvironment() as env: + try: + env.pip_install(hooks.build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_wheel({}) + log.info('Got build requires: %s', reqs) + except: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build wheel in %s', td) + try: + try: + filename = hooks.build_wheel(td, {}) + log.info('build_wheel returned %r', filename) + except: + log.info('Failure in build_wheel', exc_info=True) + return False + + if not filename.endswith('.whl'): + log.error("Filename %s doesn't have .whl extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if zipfile.is_zipfile(path): + log.info("Output file is a zip file") + else: + log.error("Output file is not a zip file") + return False + + finally: + shutil.rmtree(td) + + return True + + +def check(source_dir): + pyproject = pjoin(source_dir, 'pyproject.toml') + if isfile(pyproject): + log.info('Found pyproject.toml') + else: + log.error('Missing pyproject.toml') + return False + + try: + with open(pyproject) as f: + pyproject_data = toml_load(f) + # Ensure the mandatory data can be loaded + buildsys = pyproject_data['build-system'] + requires = buildsys['requires'] + backend = buildsys['build-backend'] + log.info('Loaded pyproject.toml') + except (TomlError, KeyError): + log.error("Invalid pyproject.toml", exc_info=True) + return False + + hooks = Pep517HookCaller(source_dir, backend) + + sdist_ok = check_build_sdist(hooks) + wheel_ok = check_build_wheel(hooks) + + if not sdist_ok: + log.warning('Sdist checks failed; scroll up to see') + if not wheel_ok: + log.warning('Wheel checks failed') + + return sdist_ok + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument('source_dir', + help="A directory containing pyproject.toml") + args = ap.parse_args(argv) + + enable_colourful_output() + + ok = check(args.source_dir) + + if ok: + print(ansi('Checks passed', 'green')) + else: + print(ansi('Checks failed', 'red')) + sys.exit(1) + +ansi_codes = { + 'reset': '\x1b[0m', + 'bold': '\x1b[1m', + 'red': '\x1b[31m', + 'green': '\x1b[32m', +} +def ansi(s, attr): + if os.name != 'nt' and sys.stdout.isatty(): + return ansi_codes[attr] + str(s) + ansi_codes['reset'] + else: + return str(s) + +if __name__ == '__main__': + main() diff --git a/pipenv/patched/notpip/_vendor/pep517/colorlog.py b/pipenv/patched/notpip/_vendor/pep517/colorlog.py new file mode 100644 index 00000000..26cf7480 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/colorlog.py @@ -0,0 +1,110 @@ +"""Nicer log formatting with colours. + +Code copied from Tornado, Apache licensed. +""" +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + +try: + import curses +except ImportError: + curses = None + +def _stderr_supports_color(): + color = False + if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): + try: + curses.setupterm() + if curses.tigetnum("colors") > 0: + color = True + except Exception: + pass + return color + +class LogFormatter(logging.Formatter): + """Log formatter with colour support + """ + DEFAULT_COLORS = { + logging.INFO: 2, # Green + logging.WARNING: 3, # Yellow + logging.ERROR: 1, # Red + logging.CRITICAL: 1, + } + + def __init__(self, color=True, datefmt=None): + r""" + :arg bool color: Enables color support. + :arg string fmt: Log message format. + It will be applied to the attributes dict of log records. The + text between ``%(color)s`` and ``%(end_color)s`` will be colored + depending on the level if color support is on. + :arg dict colors: color mappings from logging level to terminal color + code + :arg string datefmt: Datetime format. + Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. + .. versionchanged:: 3.2 + Added ``fmt`` and ``datefmt`` arguments. + """ + logging.Formatter.__init__(self, datefmt=datefmt) + self._colors = {} + if color and _stderr_supports_color(): + # The curses module has some str/bytes confusion in + # python3. Until version 3.2.3, most methods return + # bytes, but only accept strings. In addition, we want to + # output these strings with the logging module, which + # works with unicode strings. The explicit calls to + # unicode() below are harmless in python2 but will do the + # right conversion in python 3. + fg_color = (curses.tigetstr("setaf") or + curses.tigetstr("setf") or "") + if (3, 0) < sys.version_info < (3, 2, 3): + fg_color = str(fg_color, "ascii") + + for levelno, code in self.DEFAULT_COLORS.items(): + self._colors[levelno] = str(curses.tparm(fg_color, code), "ascii") + self._normal = str(curses.tigetstr("sgr0"), "ascii") + + scr = curses.initscr() + self.termwidth = scr.getmaxyx()[1] + curses.endwin() + else: + self._normal = '' + # Default width is usually 80, but too wide is worse than too narrow + self.termwidth = 70 + + def formatMessage(self, record): + l = len(record.message) + right_text = '{initial}-{name}'.format(initial=record.levelname[0], + name=record.name) + if l + len(right_text) < self.termwidth: + space = ' ' * (self.termwidth - (l + len(right_text))) + else: + space = ' ' + + if record.levelno in self._colors: + start_color = self._colors[record.levelno] + end_color = self._normal + else: + start_color = end_color = '' + + return record.message + space + start_color + right_text + end_color + +def enable_colourful_output(level=logging.INFO): + handler = logging.StreamHandler() + handler.setFormatter(LogFormatter()) + logging.root.addHandler(handler) + logging.root.setLevel(level) diff --git a/pipenv/patched/notpip/_vendor/pep517/compat.py b/pipenv/patched/notpip/_vendor/pep517/compat.py new file mode 100644 index 00000000..01c66fc7 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/compat.py @@ -0,0 +1,23 @@ +"""Handle reading and writing JSON in UTF-8, on Python 3 and 2.""" +import json +import sys + +if sys.version_info[0] >= 3: + # Python 3 + def write_json(obj, path, **kwargs): + with open(path, 'w', encoding='utf-8') as f: + json.dump(obj, f, **kwargs) + + def read_json(path): + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + +else: + # Python 2 + def write_json(obj, path, **kwargs): + with open(path, 'wb') as f: + json.dump(obj, f, encoding='utf-8', **kwargs) + + def read_json(path): + with open(path, 'rb') as f: + return json.load(f) diff --git a/pipenv/patched/notpip/_vendor/pep517/envbuild.py b/pipenv/patched/notpip/_vendor/pep517/envbuild.py new file mode 100644 index 00000000..c54d3585 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/envbuild.py @@ -0,0 +1,150 @@ +"""Build wheels/sdists by installing build deps to a temporary environment. +""" + +import os +import logging +from pipenv.patched.notpip._vendor import pytoml +import shutil +from subprocess import check_call +import sys +from sysconfig import get_paths +from tempfile import mkdtemp + +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + +def _load_pyproject(source_dir): + with open(os.path.join(source_dir, 'pyproject.toml')) as f: + pyproject_data = pytoml.load(f) + buildsys = pyproject_data['build-system'] + return buildsys['requires'], buildsys['build-backend'] + + +class BuildEnvironment(object): + """Context manager to install build deps in a simple temporary environment + + Based on code I wrote for pip, which is MIT licensed. + """ + # Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file) + # + # Permission is hereby granted, free of charge, to any person obtaining + # a copy of this software and associated documentation files (the + # "Software"), to deal in the Software without restriction, including + # without limitation the rights to use, copy, modify, merge, publish, + # distribute, sublicense, and/or sell copies of the Software, and to + # permit persons to whom the Software is furnished to do so, subject to + # the following conditions: + # + # The above copyright notice and this permission notice shall be + # included in all copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + path = None + + def __init__(self, cleanup=True): + self._cleanup = cleanup + + def __enter__(self): + self.path = mkdtemp(prefix='pep517-build-env-') + log.info('Temporary build environment: %s', self.path) + + self.save_path = os.environ.get('PATH', None) + self.save_pythonpath = os.environ.get('PYTHONPATH', None) + + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + install_dirs = get_paths(install_scheme, vars={ + 'base': self.path, + 'platbase': self.path, + }) + + scripts = install_dirs['scripts'] + if self.save_path: + os.environ['PATH'] = scripts + os.pathsep + self.save_path + else: + os.environ['PATH'] = scripts + os.pathsep + os.defpath + + if install_dirs['purelib'] == install_dirs['platlib']: + lib_dirs = install_dirs['purelib'] + else: + lib_dirs = install_dirs['purelib'] + os.pathsep + \ + install_dirs['platlib'] + if self.save_pythonpath: + os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ + self.save_pythonpath + else: + os.environ['PYTHONPATH'] = lib_dirs + + return self + + def pip_install(self, reqs): + """Install dependencies into this env by calling pip in a subprocess""" + if not reqs: + return + log.info('Calling pip to install %s', reqs) + check_call([sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--prefix', self.path] + list(reqs)) + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._cleanup and (self.path is not None) and os.path.isdir(self.path): + shutil.rmtree(self.path) + + if self.save_path is None: + os.environ.pop('PATH', None) + else: + os.environ['PATH'] = self.save_path + + if self.save_pythonpath is None: + os.environ.pop('PYTHONPATH', None) + else: + os.environ['PYTHONPATH'] = self.save_pythonpath + +def build_wheel(source_dir, wheel_dir, config_settings=None): + """Build a wheel from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str wheel_dir: Target directory to create wheel in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_wheel(config_settings) + env.pip_install(reqs) + return hooks.build_wheel(wheel_dir, config_settings) + + +def build_sdist(source_dir, sdist_dir, config_settings=None): + """Build an sdist from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str sdist_dir: Target directory to place sdist in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_sdist(config_settings) + env.pip_install(reqs) + return hooks.build_sdist(sdist_dir, config_settings) diff --git a/pipenv/patched/notpip/_vendor/pep517/wrappers.py b/pipenv/patched/notpip/_vendor/pep517/wrappers.py new file mode 100644 index 00000000..28260f32 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/wrappers.py @@ -0,0 +1,134 @@ +from contextlib import contextmanager +import os +from os.path import dirname, abspath, join as pjoin +import shutil +from subprocess import check_call +import sys +from tempfile import mkdtemp + +from . import compat + +_in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.py') + +@contextmanager +def tempdir(): + td = mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + +class UnsupportedOperation(Exception): + """May be raised by build_sdist if the backend indicates that it can't.""" + +class Pep517HookCaller(object): + """A wrapper around a source directory to be built with a PEP 517 backend. + + source_dir : The path to the source directory, containing pyproject.toml. + backend : The build backend spec, as per PEP 517, from pyproject.toml. + """ + def __init__(self, source_dir, build_backend): + self.source_dir = abspath(source_dir) + self.build_backend = build_backend + + def get_requires_for_build_wheel(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["wheel >= 0.25", "setuptools"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_wheel', { + 'config_settings': config_settings + }) + + def prepare_metadata_for_build_wheel(self, metadata_directory, config_settings=None): + """Prepare a *.dist-info folder with metadata for this project. + + Returns the name of the newly created folder. + + If the build backend defines a hook with this name, it will be called + in a subprocess. If not, the backend will be asked to build a wheel, + and the dist-info extracted from that. + """ + return self._call_hook('prepare_metadata_for_build_wheel', { + 'metadata_directory': abspath(metadata_directory), + 'config_settings': config_settings, + }) + + def build_wheel(self, wheel_directory, config_settings=None, metadata_directory=None): + """Build a wheel from this project. + + Returns the name of the newly created file. + + In general, this will call the 'build_wheel' hook in the backend. + However, if that was previously called by + 'prepare_metadata_for_build_wheel', and the same metadata_directory is + used, the previously built wheel will be copied to wheel_directory. + """ + if metadata_directory is not None: + metadata_directory = abspath(metadata_directory) + return self._call_hook('build_wheel', { + 'wheel_directory': abspath(wheel_directory), + 'config_settings': config_settings, + 'metadata_directory': metadata_directory, + }) + + def get_requires_for_build_sdist(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["setuptools >= 26"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_sdist', { + 'config_settings': config_settings + }) + + def build_sdist(self, sdist_directory, config_settings=None): + """Build an sdist from this project. + + Returns the name of the newly created file. + + This calls the 'build_sdist' backend hook in a subprocess. + """ + return self._call_hook('build_sdist', { + 'sdist_directory': abspath(sdist_directory), + 'config_settings': config_settings, + }) + + + def _call_hook(self, hook_name, kwargs): + env = os.environ.copy() + + # On Python 2, pytoml returns Unicode values (which is correct) but the + # environment passed to check_call needs to contain string values. We + # convert here by encoding using ASCII (the backend can only contain + # letters, digits and _, . and : characters, and will be used as a + # Python identifier, so non-ASCII content is wrong on Python 2 in + # any case). + if sys.version_info[0] == 2: + build_backend = self.build_backend.encode('ASCII') + else: + build_backend = self.build_backend + + env['PEP517_BUILD_BACKEND'] = build_backend + with tempdir() as td: + compat.write_json({'kwargs': kwargs}, pjoin(td, 'input.json'), + indent=2) + + # Run the hook in a subprocess + check_call([sys.executable, _in_proc_script, hook_name, td], + cwd=self.source_dir, env=env) + + data = compat.read_json(pjoin(td, 'output.json')) + if data.get('unsupported'): + raise UnsupportedOperation + return data['return_val'] + diff --git a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py index ed57821d..ac893b66 100644 --- a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py +++ b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py @@ -47,6 +47,11 @@ except ImportError: # Python 3.2 compatibility import imp as _imp +try: + FileExistsError +except NameError: + FileExistsError = OSError + from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import urllib, map, filter @@ -78,8 +83,11 @@ __import__('pipenv.patched.notpip._vendor.packaging.requirements') __import__('pipenv.patched.notpip._vendor.packaging.markers') -if (3, 0) < sys.version_info < (3, 3): - raise RuntimeError("Python 3.3 or later is required") +__metaclass__ = type + + +if (3, 0) < sys.version_info < (3, 4): + raise RuntimeError("Python 3.4 or later is required") if six.PY2: # Those builtin exceptions are only defined in Python 3 @@ -537,7 +545,7 @@ class IResourceProvider(IMetadataProvider): """List of resource names in the directory (like ``os.listdir()``)""" -class WorkingSet(object): +class WorkingSet: """A collection of active distributions on sys.path (or a similar list)""" def __init__(self, entries=None): @@ -637,13 +645,12 @@ class WorkingSet(object): distributions in the working set, otherwise only ones matching both `group` and `name` are yielded (in distribution order). """ - for dist in self: - entries = dist.get_entry_map(group) - if name is None: - for ep in entries.values(): - yield ep - elif name in entries: - yield entries[name] + return ( + entry + for dist in self + for entry in dist.get_entry_map(group).values() + if name is None or name == entry.name + ) def run_script(self, requires, script_name): """Locate distribution for `requires` and run `script_name` script""" @@ -944,7 +951,7 @@ class _ReqExtras(dict): return not req.marker or any(extra_evals) -class Environment(object): +class Environment: """Searchable snapshot of distributions on a search path""" def __init__( @@ -959,7 +966,7 @@ class Environment(object): `platform` is an optional string specifying the name of the platform that platform-specific distributions must be compatible with. If unspecified, it defaults to the current platform. `python` is an - optional string naming the desired version of Python (e.g. ``'3.3'``); + optional string naming the desired version of Python (e.g. ``'3.6'``); it defaults to the current version. You may explicitly set `platform` (and/or `python`) to ``None`` if you @@ -2087,7 +2094,12 @@ def _handle_ns(packageName, path_item): importer = get_importer(path_item) if importer is None: return None - loader = importer.find_module(packageName) + + # capture warnings due to #1111 + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + loader = importer.find_module(packageName) + if loader is None: return None module = sys.modules.get(packageName) @@ -2132,12 +2144,13 @@ def _rebuild_mod_path(orig_path, package_name, module): parts = path_parts[:-module_parts] return safe_sys_path_index(_normalize_cached(os.sep.join(parts))) - if not isinstance(orig_path, list): - # Is this behavior useful when module.__path__ is not a list? - return + new_path = sorted(orig_path, key=position_in_sys_path) + new_path = [_normalize_cached(p) for p in new_path] - orig_path.sort(key=position_in_sys_path) - module.__path__[:] = [_normalize_cached(p) for p in orig_path] + if isinstance(module.__path__, list): + module.__path__[:] = new_path + else: + module.__path__ = new_path def declare_namespace(packageName): @@ -2148,9 +2161,10 @@ def declare_namespace(packageName): if packageName in _namespace_packages: return - path, parent = sys.path, None - if '.' in packageName: - parent = '.'.join(packageName.split('.')[:-1]) + path = sys.path + parent, _, _ = packageName.rpartition('.') + + if parent: declare_namespace(parent) if parent not in _namespace_packages: __import__(parent) @@ -2161,7 +2175,7 @@ def declare_namespace(packageName): # Track what packages are namespaces, so when new path items are added, # they can be updated - _namespace_packages.setdefault(parent, []).append(packageName) + _namespace_packages.setdefault(parent or None, []).append(packageName) _namespace_packages.setdefault(packageName, []) for path_item in path: @@ -2279,7 +2293,7 @@ EGG_NAME = re.compile( ).match -class EntryPoint(object): +class EntryPoint: """Object representing an advertised importable object""" def __init__(self, name, module_name, attrs=(), extras=(), dist=None): @@ -2433,7 +2447,7 @@ def _version_from_file(lines): return safe_version(value.strip()) or None -class Distribution(object): +class Distribution: """Wrap an actual or potential sys.path entry w/metadata""" PKG_INFO = 'PKG-INFO' @@ -3027,7 +3041,10 @@ def _bypass_ensure_directory(path): dirname, filename = split(path) if dirname and filename and not isdir(dirname): _bypass_ensure_directory(dirname) - mkdir(dirname, 0o755) + try: + mkdir(dirname, 0o755) + except FileExistsError: + pass def split_sections(s): diff --git a/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py b/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py index 331a51bb..3a44fa1c 100644 --- a/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py +++ b/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py @@ -2,6 +2,8 @@ import os import errno import sys +from pipenv.patched.notpip._vendor import six + def _makedirs_31(path, exist_ok=False): try: @@ -15,8 +17,7 @@ def _makedirs_31(path, exist_ok=False): # and exists_ok considerations are disentangled. # See https://github.com/pypa/setuptools/pull/1083#issuecomment-315168663 needs_makedirs = ( - sys.version_info < (3, 2, 5) or - (3, 3) <= sys.version_info < (3, 3, 6) or + six.PY2 or (3, 4) <= sys.version_info < (3, 4, 1) ) makedirs = _makedirs_31 if needs_makedirs else os.makedirs diff --git a/pipenv/patched/notpip/_vendor/pyparsing.py b/pipenv/patched/notpip/_vendor/pyparsing.py index b5ba3d5f..455d1151 100644 --- a/pipenv/patched/notpip/_vendor/pyparsing.py +++ b/pipenv/patched/notpip/_vendor/pyparsing.py @@ -1,6 +1,6 @@ # module pyparsing.py # -# Copyright (c) 2003-2016 Paul T. McGuire +# Copyright (c) 2003-2018 Paul T. McGuire # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -25,6 +25,7 @@ __doc__ = \ """ pyparsing module - Classes and methods to define and execute parsing grammars +============================================================================= The pyparsing module is an alternative approach to creating and executing simple grammars, vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you @@ -58,10 +59,23 @@ The pyparsing module handles some of the problems that are typically vexing when - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) - quoted strings - embedded comments + + +Getting Started - +----------------- +Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing +classes inherit from. Use the docstrings for examples of how to: + - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes + - construct character word-group expressions using the L{Word} class + - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes + - use L{'+'}, L{'|'}, L{'^'}, and L{'&'} operators to combine simple expressions into more complex ones + - associate names with your parsed results using L{ParserElement.setResultsName} + - find some helpful expression short-cuts like L{delimitedList} and L{oneOf} + - find more useful common expressions in the L{pyparsing_common} namespace class """ -__version__ = "2.2.0" -__versionTime__ = "06 Mar 2017 02:06 UTC" +__version__ = "2.2.1" +__versionTime__ = "18 Sep 2018 00:49 UTC" __author__ = "Paul McGuire " import string @@ -82,6 +96,15 @@ try: except ImportError: from threading import RLock +try: + # Python 3 + from collections.abc import Iterable + from collections.abc import MutableMapping +except ImportError: + # Python 2.7 + from collections import Iterable + from collections import MutableMapping + try: from collections import OrderedDict as _OrderedDict except ImportError: @@ -940,7 +963,7 @@ class ParseResults(object): def __dir__(self): return (dir(type(self)) + list(self.keys())) -collections.MutableMapping.register(ParseResults) +MutableMapping.register(ParseResults) def col (loc,strg): """Returns current column within a string, counting newlines as line separators. @@ -1025,11 +1048,11 @@ def _trim_arity(func, maxargs=2): # special handling for Python 3.5.0 - extra deep call stack by 1 offset = -3 if system_version == (3,5,0) else -2 frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset] - return [(frame_summary.filename, frame_summary.lineno)] + return [frame_summary[:2]] def extract_tb(tb, limit=0): frames = traceback.extract_tb(tb, limit=limit) frame_summary = frames[-1] - return [(frame_summary.filename, frame_summary.lineno)] + return [frame_summary[:2]] else: extract_stack = traceback.extract_stack extract_tb = traceback.extract_tb @@ -1374,7 +1397,7 @@ class ParserElement(object): else: preloc = loc tokensStart = preloc - if self.mayIndexError or loc >= len(instring): + if self.mayIndexError or preloc >= len(instring): try: loc,tokens = self.parseImpl( instring, preloc, doActions ) except IndexError: @@ -1408,7 +1431,6 @@ class ParserElement(object): self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), modal=self.modalResults ) - if debugging: #~ print ("Matched",self,"->",retTokens.asList()) if (self.debugActions[1] ): @@ -3242,7 +3264,7 @@ class ParseExpression(ParserElement): if isinstance( exprs, basestring ): self.exprs = [ ParserElement._literalStringClass( exprs ) ] - elif isinstance( exprs, collections.Iterable ): + elif isinstance( exprs, Iterable ): exprs = list(exprs) # if sequence of strings provided, wrap with Literal if all(isinstance(expr, basestring) for expr in exprs): @@ -4393,7 +4415,7 @@ def traceParseAction(f): @traceParseAction def remove_duplicate_chars(tokens): - return ''.join(sorted(set(''.join(tokens))) + return ''.join(sorted(set(''.join(tokens)))) wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) print(wds.parseString("slkdjs sld sldd sdlf sdljf")) @@ -4583,7 +4605,7 @@ def oneOf( strs, caseless=False, useRegex=True ): symbols = [] if isinstance(strs,basestring): symbols = strs.split() - elif isinstance(strs, collections.Iterable): + elif isinstance(strs, Iterable): symbols = list(strs) else: warnings.warn("Invalid argument to oneOf, expected string or iterable", @@ -4734,7 +4756,7 @@ stringEnd = StringEnd().setName("stringEnd") _escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1]) _escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16))) _escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8))) -_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(printables, excludeChars=r'\]', exact=1) | Regex(r"\w", re.UNICODE) +_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1) _charRange = Group(_singleChar + Suppress("-") + _singleChar) _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" diff --git a/pipenv/patched/notpip/_vendor/pytoml/parser.py b/pipenv/patched/notpip/_vendor/pytoml/parser.py index e03a03fb..9f94e923 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/parser.py +++ b/pipenv/patched/notpip/_vendor/pytoml/parser.py @@ -223,8 +223,8 @@ _float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]? _datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') _basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') -_litstr_re = re.compile(r"[^'\000-\037]*") -_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*") +_litstr_re = re.compile(r"[^'\000\010\012-\037]*") +_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") def _p_value(s, object_pairs_hook): pos = s.pos() diff --git a/pipenv/patched/notpip/_vendor/requests/__init__.py b/pipenv/patched/notpip/_vendor/requests/__init__.py index 8e7576c1..af3d4b1d 100644 --- a/pipenv/patched/notpip/_vendor/requests/__init__.py +++ b/pipenv/patched/notpip/_vendor/requests/__init__.py @@ -91,7 +91,7 @@ except (AssertionError, ValueError): RequestsDependencyWarning) # Attempt to enable urllib3's SNI support, if possible -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS if not WINDOWS: try: from pipenv.patched.notpip._vendor.urllib3.contrib import pyopenssl diff --git a/pipenv/patched/notpip/_vendor/vendor.txt b/pipenv/patched/notpip/_vendor/vendor.txt index b9854e9a..9389dd94 100644 --- a/pipenv/patched/notpip/_vendor/vendor.txt +++ b/pipenv/patched/notpip/_vendor/vendor.txt @@ -9,14 +9,15 @@ msgpack-python==0.5.6 lockfile==0.12.2 progress==1.4 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==17.1 -pyparsing==2.2.0 -pytoml==0.1.16 +packaging==18.0 +pep517==0.2 +pyparsing==2.2.1 +pytoml==0.1.19 retrying==1.3.3 requests==2.19.1 chardet==3.0.4 idna==2.7 urllib3==1.23 - certifi==2018.4.16 -setuptools==39.2.0 + certifi==2018.8.24 +setuptools==40.4.3 webencodings==0.5.1 diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index 3ad4c1e9..f73ed5a6 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -4,4 +4,4 @@ crayons==0.1.2 pipfile==0.0.2 pip-tools==3.1.0 prettytoml==0.3 -pip==18.0 +pip==18.1 diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py index de9b4353..c466ef04 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -1,49 +1,55 @@ # -*- coding=utf-8 -*- -import importlib -from pip_shims import pip_version -import pkg_resources +__all__ = [ + "InstallRequirement", + "parse_requirements", + "RequirementSet", + "user_cache_dir", + "FAVORITE_HASH", + "is_file_url", + "url_to_path", + "PackageFinder", + "FormatControl", + "Wheel", + "Command", + "cmdoptions", + "get_installed_distributions", + "PyPI", + "SafeFileCache", + "InstallationError", + "parse_version", + "pip_version", + "install_req_from_editable", + "install_req_from_line", + "user_cache_dir" +] -def do_import(module_path, subimport=None, old_path=None, vendored_name=None): - old_path = old_path or module_path - prefix = vendored_name if vendored_name else "pip" - prefixes = ["{0}._internal".format(prefix), "{0}".format(prefix)] - paths = [module_path, old_path] - search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] - package = subimport if subimport else None - for to_import in search_order: - if not subimport: - to_import, _, package = to_import.rpartition(".") - try: - imported = importlib.import_module(to_import) - except ImportError: - continue - else: - return getattr(imported, package) - - -InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name="notpip") -parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name="notpip") -RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name="notpip") -user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name="notpip") -FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name="notpip") -is_file_url = do_import('download', 'is_file_url', vendored_name="notpip") -url_to_path = do_import('download', 'url_to_path', vendored_name="notpip") -PackageFinder = do_import('index', 'PackageFinder', vendored_name="notpip") -FormatControl = do_import('index', 'FormatControl', vendored_name="notpip") -Wheel = do_import('wheel', 'Wheel', vendored_name="notpip") -Command = do_import('cli.base_command', 'Command', old_path='basecommand', vendored_name="notpip") -cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions', vendored_name="notpip") -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name="notpip") -PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') -SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') -InstallationError = do_import('exceptions', 'InstallationError', vendored_name='notpip') +from pipenv.vendor.appdirs import user_cache_dir +from pip_shims.shims import ( + InstallRequirement, + parse_requirements, + RequirementSet, + FAVORITE_HASH, + is_file_url, + url_to_path, + PackageFinder, + FormatControl, + Wheel, + Command, + cmdoptions, + get_installed_distributions, + PyPI, + SafeFileCache, + InstallationError, + parse_version, + pip_version, +) # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if pkg_resources.parse_version(pip_version) < pkg_resources.parse_version('18.1'): +if parse_version(pip_version) < parse_version('18.1'): install_req_from_line = InstallRequirement.from_line install_req_from_editable = InstallRequirement.from_editable else: - install_req_from_line = do_import('req.constructors', 'install_req_from_line', vendored_name="notpip") - install_req_from_editable = do_import('req.constructors', 'install_req_from_editable', vendored_name="notpip") - + from pip_shims.shims import ( + install_req_from_editable, install_req_from_line + ) diff --git a/pipenv/project.py b/pipenv/project.py index 280a6f8b..dcf9b417 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -639,8 +639,9 @@ class Project(object): def create_pipfile(self, python=None): """Creates the Pipfile, filled with juicy defaults.""" - from .patched.notpip._internal import ConfigOptionParser - from .patched.notpip._internal.cmdoptions import make_option_group, index_group + from .vendor.pip_shims.shims import ( + ConfigOptionParser, make_option_group, index_group + ) config_parser = ConfigOptionParser(name=self.name) config_parser.add_option_group(make_option_group(index_group, config_parser)) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index b82088a2..b5ea41d9 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -97,9 +97,16 @@ def main(): import warnings from pipenv.vendor.vistir.compat import ResourceWarning warnings.simplefilter("ignore", category=ResourceWarning) - from pipenv.vendor import colorama - colorama.init() + import io + import six + if six.PY3: + sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') + sys.stderr = io.TextIOWrapper(sys.stderr.buffer,encoding='utf8') + else: + from pipenv._compat import force_encoding + force_encoding() os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") + os.environ["PYTHONIOENCODING"] = str("utf-8") parser = get_parser() parsed, remaining = parser.parse_known_args() # sys.argv = remaining @@ -110,4 +117,6 @@ def main(): if __name__ == "__main__": _patch_path() + from pipenv.vendor import colorama + colorama.init() main() diff --git a/pipenv/utils.py b/pipenv/utils.py index 7c78de34..893ed88f 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -211,10 +211,10 @@ def actually_resolve_deps( pre, req_dir=None, ): - from .patched.notpip._internal import basecommand - from .patched.notpip._internal.req import parse_requirements - from .patched.notpip._internal.exceptions import DistributionNotFound - from .patched.notpip._vendor.requests.exceptions import HTTPError + from .vendor.pip_shims.shims import ( + Command, parse_requirements, DistributionNotFound + ) + from .vendor.requests.exceptions import HTTPError from pipenv.patched.piptools.resolver import Resolver from pipenv.patched.piptools.repositories.pypi import PyPIRepository from pipenv.patched.piptools.scripts.compile import get_pip_command @@ -223,7 +223,7 @@ def actually_resolve_deps( from .vendor.requirementslib.models.requirements import Requirement from .vendor.vistir.path import create_tracked_tempdir, create_tracked_tempfile - class PipCommand(basecommand.Command): + class PipCommand(Command): """Needed for pip-tools.""" name = "PipCommand" @@ -338,6 +338,7 @@ def venv_resolve_deps( from .vendor.pexpect.exceptions import EOF, TIMEOUT from .vendor import delegator from . import resolver + from ._compat import decode_output import json if not deps: @@ -380,14 +381,14 @@ def venv_resolve_deps( break _out = c.subprocess.before if _out is not None: - _out = to_native_string("{0}".format(_out)) + _out = decode_output("{0}".format(_out)) out += _out - sp.text = to_native_string("Locking... {0}".format(_out[:100])) - if environments.is_verbose(): - if _out is not None: - sp._hide_cursor() - sp.write(_out.rstrip()) - sp._show_cursor() + sp.text = to_native_string("{0}".format(_out[:100])) + if environments.is_verbose(): + if _out is not None: + sp._hide_cursor() + sp.write(_out.rstrip()) + sp._show_cursor() c.block() if c.return_code != 0: sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( @@ -423,7 +424,7 @@ def resolve_deps( """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip. """ - from .patched.notpip._vendor.requests.exceptions import ConnectionError + from .vendor.requests.exceptions import ConnectionError from .vendor.requirementslib.models.requirements import Requirement index_lookup = {} @@ -637,9 +638,8 @@ def is_editable(pipfile_entry): def is_installable_file(path): """Determine if a path can potentially be installed""" - from .patched.notpip._internal.utils.misc import is_installable_dir + from .vendor.pip_shims.shims import is_installable_dir, is_archive_file from .patched.notpip._internal.utils.packaging import specifiers - from .patched.notpip._internal.download import is_archive_file from ._compat import Path if hasattr(path, "keys") and any( @@ -691,7 +691,7 @@ def is_file(package): def pep440_version(version): """Normalize version to PEP 440 standards""" - from .patched.notpip._internal.index import parse_version + from .vendor.pip_shims.shims import parse_version # Use pip built-in version parser. return str(parse_version(version)) @@ -1149,7 +1149,7 @@ def translate_markers(pipfile_entry): """ if not isinstance(pipfile_entry, Mapping): raise TypeError("Entry is not a pipfile formatted mapping.") - from notpip._vendor.distlib.markers import DEFAULT_CONTEXT as marker_context + from .vendor.distlib.markers import DEFAULT_CONTEXT as marker_context from .vendor.packaging.markers import Marker from .vendor.vistir.misc import dedup diff --git a/pipenv/vendor/backports/__init__.py b/pipenv/vendor/backports/__init__.py index 3cd30963..0c64b4c1 100644 --- a/pipenv/vendor/backports/__init__.py +++ b/pipenv/vendor/backports/__init__.py @@ -1,7 +1,5 @@ -# See https://pypi.python.org/pypi/backports - -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) from . import weakref from . import enum from . import shutil_get_terminal_size +from . import functools_lru_cache diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py index 4bf72946..6ba28bbb 100644 --- a/pipenv/vendor/dotenv/main.py +++ b/pipenv/vendor/dotenv/main.py @@ -95,6 +95,13 @@ class DotEnv(): for k, v in self.dict().items(): if k in os.environ and not override: continue + # With Python 2 on Windows, ensuree environment variables are + # system strings to avoid "TypeError: environment can only contain + # strings" in Python's subprocess module. + if sys.version_info.major < 3 and sys.platform == 'win32': + from pipenv.utils import fs_str + k = fs_str(k) + v = fs_str(v) os.environ[k] = v return True diff --git a/pipenv/vendor/modutil.LICENSE b/pipenv/vendor/modutil.LICENSE deleted file mode 100644 index f680f071..00000000 --- a/pipenv/vendor/modutil.LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2018, Brett Cannon -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/passa/internals/dependencies.py b/pipenv/vendor/passa/internals/dependencies.py index 6ce97aad..358cc33b 100644 --- a/pipenv/vendor/passa/internals/dependencies.py +++ b/pipenv/vendor/passa/internals/dependencies.py @@ -216,7 +216,7 @@ def _read_requires_python(metadata): def _get_dependencies_from_pip(ireq, sources): - """Retrieves dependencies for the requirement from pip internals. + """Retrieves dependencies for the requirement from pipenv.patched.notpip internals. The current strategy is to try the followings in order, returning the first successful result. diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py index 70fb0d58..22bf130f 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import sys -__version__ = '0.3.1' +__version__ = '0.3.2' from . import shims diff --git a/pipenv/vendor/pip_shims/shims.py b/pipenv/vendor/pip_shims/shims.py index 7b81a608..34de6906 100644 --- a/pipenv/vendor/pip_shims/shims.py +++ b/pipenv/vendor/pip_shims/shims.py @@ -43,6 +43,9 @@ class _shims(object): "pip_shims.utils": utils } self.pip_version = getattr(self._modules["pip"], "__version__") + version_types = ["post", "pre", "dev", "rc"] + if any(post in self.pip_version.rsplit(".")[-1] for post in version_types): + self.pip_version, _, _ = self.pip_version.rpartition(".") self.parsed_pip_version = self._parse(self.pip_version) self._contextmanagers = ("RequirementTracker",) self._moves = { @@ -140,6 +143,7 @@ class _shims(object): ("wheel.WheelCache", "7", "9.0.3") ), "WheelBuilder": ("wheel.WheelBuilder", "7.0.0", "9999"), + "PyPI": ("models.index.PyPI", "7.0.0", "9999"), } def _ensure_methods(self, cls, classname, *methods): diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index d800f926..60192826 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.5' +__version__ = '1.1.6' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 4a4c50da..20b2c196 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -27,6 +27,7 @@ from ..utils import ( path_is_known_executable, unnest, ) +from .python import PythonVersion @attr.s diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index a9d6105b..1595a963 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -14,6 +14,8 @@ from vistir.compat import Path from ..utils import ( ensure_path, optional_instance_of, + get_python_version, + filter_pythons, unnest, ) from .mixins import BaseFinder, BasePath diff --git a/pipenv/vendor/requests/LICENSE b/pipenv/vendor/requests/LICENSE index 2e68b82e..841c6023 100644 --- a/pipenv/vendor/requests/LICENSE +++ b/pipenv/vendor/requests/LICENSE @@ -4,7 +4,7 @@ Copyright 2018 Kenneth Reitz you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 3103580e..d72542c4 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -132,7 +132,7 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pip format""" + """Cleans VCS uris from pipenv.patched.notpip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: diff --git a/pipenv/vendor/shutil_backports/__init__.py b/pipenv/vendor/shutil_backports/__init__.py deleted file mode 100644 index fa12816e..00000000 --- a/pipenv/vendor/shutil_backports/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -__title__ = "shutil_backports" -__version__ = "0.1.0" -__license__ = "MIT" -__author__ = "Christopher Rosell" -__copyright__ = "Copyright 2014 Christopher Rosell" - -__all__ = ["get_terminal_size"] - -from .get_terminal_size import * diff --git a/pipenv/vendor/shutil_backports/get_terminal_size.py b/pipenv/vendor/shutil_backports/get_terminal_size.py deleted file mode 100644 index f1336e58..00000000 --- a/pipenv/vendor/shutil_backports/get_terminal_size.py +++ /dev/null @@ -1,100 +0,0 @@ -"""This is a backport of shutil.get_terminal_size from Python 3.3. - -The original implementation is in C, but here we use the ctypes and -fcntl modules to create a pure Python version of os.get_terminal_size. -""" - -import os -import struct -import sys - -from collections import namedtuple - -__all__ = ["get_terminal_size"] - - -terminal_size = namedtuple("terminal_size", "columns lines") - -try: - from ctypes import windll, create_string_buffer - - _handles = { - 0: windll.kernel32.GetStdHandle(-10), - 1: windll.kernel32.GetStdHandle(-11), - 2: windll.kernel32.GetStdHandle(-12), - } - - def _get_terminal_size(fd): - columns = lines = 0 - - try: - handle = _handles[fd] - csbi = create_string_buffer(22) - res = windll.kernel32.GetConsoleScreenBufferInfo(handle, csbi) - if res: - res = struct.unpack("hhhhHhhhhhh", csbi.raw) - left, top, right, bottom = res[5:9] - columns = right - left + 1 - lines = bottom - top + 1 - except Exception: - pass - - return columns, lines - -except ImportError: - import fcntl - import termios - - def _get_terminal_size(fd): - try: - res = fcntl.ioctl(fd, termios.TIOCGWINSZ, b"\x00" * 4) - lines, columns = struct.unpack("hh", res) - except Exception: - columns = lines = 0 - - return columns, lines - - -def get_terminal_size(fallback=(80, 24)): - """Get the size of the terminal window. - - For each of the two dimensions, the environment variable, COLUMNS - and LINES respectively, is checked. If the variable is defined and - the value is a positive integer, it is used. - - When COLUMNS or LINES is not defined, which is the common case, - the terminal connected to sys.__stdout__ is queried - by invoking os.get_terminal_size. - - If the terminal size cannot be successfully queried, either because - the system doesn't support querying, or because we are not - connected to a terminal, the value given in fallback parameter - is used. Fallback defaults to (80, 24) which is the default - size used by many terminal emulators. - - The value returned is a named tuple of type os.terminal_size. - """ - # Attempt to use the environment first - try: - columns = int(os.environ["COLUMNS"]) - except (KeyError, ValueError): - columns = 0 - - try: - lines = int(os.environ["LINES"]) - except (KeyError, ValueError): - lines = 0 - - # Only query if necessary - if columns <= 0 or lines <= 0: - try: - columns, lines = _get_terminal_size(sys.__stdout__.fileno()) - except (NameError, OSError): - pass - - # Use fallback as last resort - if columns <= 0 and lines <= 0: - columns, lines = fallback - - return terminal_size(columns, lines) - diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 091f27d8..35a32eb6 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -42,11 +42,12 @@ shutilwhich==1.1.0 toml==0.10.0 cached-property==1.4.3 vistir==0.2.2 -pip-shims==0.3.1 +pip-shims==0.3.2 ptyprocess==0.6.0 enum34==1.1.6 yaspin==0.14.0 cerberus==1.2 git+https://github.com/sarugaku/passa.git@master#egg=passa cursor==1.2.0 +resolvelib==0.2.2 backports.functools_lru_cache==1.5 diff --git a/pipenv/vendor/vendor_pip.txt b/pipenv/vendor/vendor_pip.txt index b9854e9a..9389dd94 100644 --- a/pipenv/vendor/vendor_pip.txt +++ b/pipenv/vendor/vendor_pip.txt @@ -9,14 +9,15 @@ msgpack-python==0.5.6 lockfile==0.12.2 progress==1.4 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==17.1 -pyparsing==2.2.0 -pytoml==0.1.16 +packaging==18.0 +pep517==0.2 +pyparsing==2.2.1 +pytoml==0.1.19 retrying==1.3.3 requests==2.19.1 chardet==3.0.4 idna==2.7 urllib3==1.23 - certifi==2018.4.16 -setuptools==39.2.0 + certifi==2018.8.24 +setuptools==40.4.3 webencodings==0.5.1 diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 4d8a3191..59b97ca0 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -118,15 +118,19 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): """ from .spin import create_spinner - if nospin is False: - try: - import yaspin - except ImportError: + has_yaspin = False + try: + import yaspin + except ImportError: + if not nospin: raise RuntimeError( "Failed to import spinner! Reinstall vistir with command:" " pip install --upgrade vistir[spinner]" ) + else: + spinner_name = "" else: + has_yaspin = True spinner_name = "" if not start_text and nospin is False: start_text = "Running..." @@ -135,6 +139,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): text=start_text, handler_map=handler_map, nospin=nospin, + use_yaspin=has_yaspin ) as _spinner: yield _spinner diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 7d400a4c..7342bc97 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -489,3 +489,36 @@ try: locale_encoding = locale.getdefaultencoding()[1] or "ascii" except Exception: locale_encoding = "ascii" + + +def getpreferredencoding(): + import locale + # Borrowed from Invoke + # (see https://github.com/pyinvoke/invoke/blob/93af29d/invoke/runners.py#L881) + _encoding = locale.getpreferredencoding(False) + if six.PY2 and not sys.platform == "win32": + _default_encoding = locale.getdefaultlocale()[1] + if _default_encoding is not None: + _encoding = _default_encoding + return _encoding + + +PREFERRED_ENCODING = getpreferredencoding() + + +def decode_for_output(output): + """Given a string, decode it for output to a terminal + + :param str output: A string to print to a terminal + :return: A re-encoded string using the preferred encoding + :rtype: str + """ + + if not isinstance(output, six.string_types): + return output + try: + output = output.encode(PREFERRED_ENCODING) + except AttributeError: + pass + output = output.decode(PREFERRED_ENCODING) + return output diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index 57a90277..6b6e498f 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -33,7 +33,8 @@ CLEAR_LINE = chr(27) + "[K" class DummySpinner(object): def __init__(self, text="", **kwargs): colorama.init() - self.text = to_native_string(text) + from .misc import decode_for_output + self.text = to_native_string(decode_for_output(text)) self.stdout = kwargs.get("stdout", sys.stdout) self.stderr = kwargs.get("stderr", sys.stderr) self.out_buff = StringIO() @@ -44,10 +45,11 @@ class DummySpinner(object): return self def __exit__(self, exc_type, exc_val, traceback): - if not exc_type: - self.ok(text=None) - else: - self.write_err(traceback) + if exc_type: + import traceback + from .misc import decode_for_output + self.write_err(decode_for_output(traceback.format_exception(traceback))) + self._close_output_buffer() return False def __getattr__(self, k): @@ -60,33 +62,42 @@ class DummySpinner(object): else: return retval + def _close_output_buffer(self): + if self.out_buff and not self.out_buff.closed: + try: + self.out_buff.close() + except Exception: + pass + def fail(self, exitcode=1, text="FAIL"): + from .misc import decode_for_output if text and text != "None": - self.write_err(text) - if self.out_buff: - self.out_buff.close() - raise SystemExit(exitcode, text) + self.write_err(decode_for_output(text)) + self._close_output_buffer() def ok(self, text="OK"): if text and text != "None": self.stderr.write(self.text) - if self.out_buff: - self.out_buff.close() + self._close_output_buffer() return 0 def write(self, text=None): + from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - self.stdout.write(to_native_string("\r")) - line = to_native_string("{0}\n".format(text)) + text = decode_for_output(text) + self.stdout.write(decode_for_output("\r")) + line = decode_for_output("{0}\n".format(text)) self.stdout.write(line) self.stdout.write(CLEAR_LINE) def write_err(self, text=None): + from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - self.stderr.write(to_native_string("\r")) - line = to_native_string("{0}\n".format(text)) + text = decode_for_output(text) + self.stderr.write(decode_for_output("\r")) + line = decode_for_output("{0}\n".format(text)) self.stderr.write(line) self.stderr.write(CLEAR_LINE) diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index d01fb98e..06b8b621 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -16,6 +16,9 @@ import sys import threading import time +import colorama +import cursor + from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS @@ -23,6 +26,9 @@ from .helpers import to_unicode from .termcolor import colored +colorama.init() + + class Yaspin(object): """Implements a context manager that spawns a thread to write spinner frames into a tty (stdout) during @@ -369,11 +375,14 @@ class Yaspin(object): # SIGKILL cannot be caught or ignored, and the receiving # process cannot perform any clean-up upon receiving this # signal. - if signal.SIGKILL in self._sigmap.keys(): - raise ValueError( - "Trying to set handler for SIGKILL signal. " - "SIGKILL cannot be cought or ignored in POSIX systems." - ) + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass for sig, sig_handler in iteritems(self._sigmap): # A handler for a particular signal, once set, remains @@ -521,14 +530,12 @@ class Yaspin(object): @staticmethod def _hide_cursor(): - sys.stdout.write("\033[?25l") - sys.stdout.flush() + cursor.hide() @staticmethod def _show_cursor(): - sys.stdout.write("\033[?25h") - sys.stdout.flush() + cursor.show() @staticmethod def _clear_line(): - sys.stdout.write("\033[K") + sys.stdout.write(chr(27) + "[K") diff --git a/pytest.ini b/pytest.ini index 48dfab02..2bfca079 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] addopts = -ra -n auto -testpaths = tests/ +testpaths = tests ; Add vendor and patched in addition to the default list of ignored dirs norecursedirs = .* build dist CVS _darcs {arch} *.egg vendor patched news tasks docs filterwarnings = diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 38ec032f..4d968a7d 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -20,6 +20,7 @@ LIBRARY_DIRNAMES = { 'requirements-parser': 'requirements', 'backports.shutil_get_terminal_size': 'backports/shutil_get_terminal_size', 'backports.weakref': 'backports/weakref', + 'backports.functools_lru_cache': 'backports/functools_lru_cache', 'shutil_backports': 'backports/shutil_get_terminal_size', 'python-dotenv': 'dotenv', 'pip-tools': 'piptools', @@ -49,7 +50,8 @@ HARDCODED_LICENSE_URLS = { 'master/LICENSE', 'requirementslib': 'https://github.com/techalchemy/requirementslib/raw/master/LICENSE', 'distlib': 'https://github.com/vsajip/distlib/raw/master/LICENSE.txt', - 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt' + 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt', + 'pyparsing': 'https://raw.githubusercontent.com/pyparsing/pyparsing/master/LICENSE', } FILE_WHITE_LIST = ( @@ -78,7 +80,7 @@ LIBRARY_RENAMES = { def drop_dir(path): if path.exists() and path.is_dir(): - shutil.rmtree(str(path)) + shutil.rmtree(str(path), ignore_errors=True) def remove_all(paths): @@ -112,8 +114,6 @@ def clean_vendor(ctx, vendor_dir): for item in vendor_dir.iterdir(): if item.is_dir(): shutil.rmtree(str(item)) - elif "LICENSE" in item.name or "COPYING" in item.name: - continue elif item.name not in FILE_WHITE_LIST: item.unlink() else: @@ -332,6 +332,7 @@ def post_install_cleanup(ctx, vendor_dir): # Cleanup setuptools unneeded parts drop_dir(vendor_dir / 'bin') drop_dir(vendor_dir / 'tests') + remove_all(vendor_dir.glob('toml.py')) def vendor(ctx, vendor_dir, package=None, rewrite=True): @@ -355,6 +356,10 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): log("Removing scandir library files...") remove_all(vendor_dir.glob('*.so')) + drop_dir(vendor_dir / 'setuptools') + drop_dir(vendor_dir / 'pkg_resources' / '_vendor') + drop_dir(vendor_dir / 'pkg_resources' / 'extern') + drop_dir(vendor_dir / 'bin') # Global import rewrites log('Renaming specified libs...') diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch index 654a3da9..fce6ae89 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch @@ -1,20 +1,8 @@ diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py -index c9804ab5..89c95c45 100644 +index f3b9b5b4..182c1c88 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py -@@ -10,10 +10,7 @@ import sysconfig - import warnings - from collections import OrderedDict - --try: -- import pipenv.patched.notpip._internal.utils.glibc --except ImportError: -- import pipenv.patched.notpip.utils.glibc -+import pipenv.patched.notpip._internal.utils.glibc - - logger = logging.getLogger(__name__) - -@@ -157,7 +154,7 @@ def is_manylinux1_compatible(): +@@ -159,7 +159,7 @@ def is_manylinux1_compatible(): pass # Check glibc version. CentOS 5 uses glibc 2.5. diff --git a/tasks/vendoring/patches/patched/pip18.patch b/tasks/vendoring/patches/patched/pip18.patch index 558c28e8..150ee32f 100644 --- a/tasks/vendoring/patches/patched/pip18.patch +++ b/tasks/vendoring/patches/patched/pip18.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/patched/pip/_internal/download.py b/pipenv/patched/pip/_internal/download.py -index 96f3b65c..3fb4ebef 100644 +index 96f3b65c..cc5b3d15 100644 --- a/pipenv/patched/pip/_internal/download.py +++ b/pipenv/patched/pip/_internal/download.py @@ -19,6 +19,7 @@ from pip._vendor.lockfile import LockError @@ -19,43 +19,11 @@ index 96f3b65c..3fb4ebef 100644 "python": platform.python_version(), "implementation": { "name": platform.python_implementation(), -@@ -322,7 +323,7 @@ class InsecureHTTPAdapter(HTTPAdapter): - conn.ca_certs = None - - --class PipSession(requests.Session): -+class PipSession(Session): - - timeout = None - -@@ -752,7 +753,7 @@ def _copy_dist_from_dir(link_path, location): - - # build an sdist - setup_py = 'setup.py' -- sdist_args = [sys.executable] -+ sdist_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable)] - sdist_args.append('-c') - sdist_args.append(SETUPTOOLS_SHIM % setup_py) - sdist_args.append('sdist') diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py -index 8c0ec82c..ad00ba04 100644 +index 8c2f24f1..cdd48874 100644 --- a/pipenv/patched/pip/_internal/index.py +++ b/pipenv/patched/pip/_internal/index.py -@@ -58,11 +58,12 @@ logger = logging.getLogger(__name__) - - class InstallationCandidate(object): - -- def __init__(self, project, version, location): -+ def __init__(self, project, version, location, requires_python=None): - self.project = project - self.version = parse_version(version) - self.location = location - self._key = (self.project, self.version, self.location) -+ self.requires_python = requires_python - - def __repr__(self): - return "".format( -@@ -168,6 +169,9 @@ class PackageFinder(object): +@@ -246,6 +246,9 @@ class PackageFinder(object): # The Session we'll use to make requests self.session = session @@ -65,7 +33,7 @@ index 8c0ec82c..ad00ba04 100644 # The valid tags to check potential found wheel candidates against self.valid_tags = get_supported( versions=versions, -@@ -220,6 +224,24 @@ class PackageFinder(object): +@@ -298,6 +301,25 @@ class PackageFinder(object): ) self.dependency_links.extend(links) @@ -86,11 +54,12 @@ index 8c0ec82c..ad00ba04 100644 + current_list.append(link) + + return extras ++ + @staticmethod def _sort_locations(locations, expand_dir=False): """ -@@ -272,7 +294,7 @@ class PackageFinder(object): +@@ -350,7 +372,7 @@ class PackageFinder(object): return files, urls @@ -99,7 +68,7 @@ index 8c0ec82c..ad00ba04 100644 """ Function used to generate link sort key for link tuples. The greater the return value, the more preferred it is. -@@ -292,14 +314,19 @@ class PackageFinder(object): +@@ -370,14 +392,19 @@ class PackageFinder(object): if candidate.location.is_wheel: # can raise InvalidWheelFilename wheel = Wheel(candidate.location.filename) @@ -121,7 +90,19 @@ index 8c0ec82c..ad00ba04 100644 if wheel.build_tag is not None: match = re.match(r'^(\d+)(.*)$', wheel.build_tag) build_tag_groups = match.groups() -@@ -484,7 +511,7 @@ class PackageFinder(object): +@@ -528,7 +555,10 @@ class PackageFinder(object): + + page_versions = [] + for page in self._get_pages(url_locations, project_name): +- logger.debug('Analyzing links from page %s', page.url) ++ try: ++ logger.debug('Analyzing links from page %s', page.url) ++ except AttributeError: ++ continue + with indent_log(): + page_versions.extend( + self._package_versions(page.iter_links(), search) +@@ -562,7 +592,7 @@ class PackageFinder(object): dependency_versions ) @@ -130,19 +111,19 @@ index 8c0ec82c..ad00ba04 100644 """Try to find a Link matching req Expects req, an InstallRequirement and upgrade, a boolean -@@ -594,8 +621,9 @@ class PackageFinder(object): +@@ -672,7 +702,10 @@ class PackageFinder(object): continue seen.add(location) - page = self._get_page(location) -- if page is None: + try: + page = self._get_page(location) -+ except requests.HTTPError as e: ++ except requests.HTTPError: ++ continue + if page is None: continue - yield page -@@ -631,7 +659,7 @@ class PackageFinder(object): +@@ -709,7 +742,7 @@ class PackageFinder(object): logger.debug('Skipping link %s; %s', link, reason) self.logged_links.add(link) @@ -151,7 +132,7 @@ index 8c0ec82c..ad00ba04 100644 """Return an InstallationCandidate or None""" version = None if link.egg_fragment: -@@ -647,12 +675,12 @@ class PackageFinder(object): +@@ -725,12 +758,12 @@ class PackageFinder(object): link, 'unsupported archive format: %s' % ext, ) return @@ -166,7 +147,7 @@ index 8c0ec82c..ad00ba04 100644 self._log_skipped_link(link, 'macosx10 one') return if ext == wheel_ext: -@@ -666,7 +694,7 @@ class PackageFinder(object): +@@ -744,7 +777,7 @@ class PackageFinder(object): link, 'wrong project name (not %s)' % search.supplied) return @@ -175,7 +156,7 @@ index 8c0ec82c..ad00ba04 100644 self._log_skipped_link( link, 'it is not compatible with this Python') return -@@ -702,14 +730,14 @@ class PackageFinder(object): +@@ -780,14 +813,14 @@ class PackageFinder(object): link.filename, link.requires_python) support_this_python = True @@ -191,13 +172,30 @@ index 8c0ec82c..ad00ba04 100644 + return InstallationCandidate(search.supplied, version, link, link.requires_python) def _get_page(self, link): - return HTMLPage.get_page(link, session=self.session) + return _get_html_page(link, session=self.session) +diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py +index c736de6c..a78566c1 100644 +--- a/pipenv/patched/pip/_internal/models/candidate.py ++++ b/pipenv/patched/pip/_internal/models/candidate.py +@@ -7,10 +7,11 @@ class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + +- def __init__(self, project, version, location): ++ def __init__(self, project, version, location, requires_python=None): + self.project = project + self.version = parse_version(version) + self.location = location ++ self.requires_python = requires_python + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py -index 7740c284..b6e946d8 100644 +index 104bea33..ecf78b9a 100644 --- a/pipenv/patched/pip/_internal/operations/prepare.py +++ b/pipenv/patched/pip/_internal/operations/prepare.py @@ -17,7 +17,7 @@ from pip._internal.exceptions import ( - ) + from pip._internal.utils.compat import expanduser from pip._internal.utils.hashes import MissingHashes from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import display_path, normalize_path @@ -206,8 +204,8 @@ index 7740c284..b6e946d8 100644 logger = logging.getLogger(__name__) @@ -123,7 +123,11 @@ class IsSDist(DistAbstraction): - "Installing build dependencies" - ) + " and ".join(map(repr, sorted(missing))) + ) - self.req.run_egg_info() + try: @@ -237,10 +235,10 @@ index 7740c284..b6e946d8 100644 # We can't hit this spot and have populate_link return None. diff --git a/pipenv/patched/pip/_internal/pep425tags.py b/pipenv/patched/pip/_internal/pep425tags.py -index 0b5c7832..bea31585 100644 +index ab1a0298..763c0a24 100644 --- a/pipenv/patched/pip/_internal/pep425tags.py +++ b/pipenv/patched/pip/_internal/pep425tags.py -@@ -10,7 +10,10 @@ import sysconfig +@@ -10,7 +10,11 @@ import sysconfig import warnings from collections import OrderedDict @@ -249,81 +247,80 @@ index 0b5c7832..bea31585 100644 + import pip._internal.utils.glibc +except ImportError: + import pip.utils.glibc ++ + from pip._internal.utils.compat import get_extension_suffixes logger = logging.getLogger(__name__) - diff --git a/pipenv/patched/pip/_internal/req/req_install.py b/pipenv/patched/pip/_internal/req/req_install.py -index 462c80aa..d039adc8 100644 +index c2624fee..ee75acd6 100644 --- a/pipenv/patched/pip/_internal/req/req_install.py +++ b/pipenv/patched/pip/_internal/req/req_install.py -@@ -615,7 +615,7 @@ class InstallRequirement(object): +@@ -452,7 +452,8 @@ class InstallRequirement(object): with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py - base_cmd = [sys.executable, '-c', script] -+ base_cmd = [os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', script] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ base_cmd = [sys_executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] -@@ -797,7 +797,7 @@ class InstallRequirement(object): +@@ -613,10 +614,11 @@ class InstallRequirement(object): + + with indent_log(): + # FIXME: should we do --install-headers here too? ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) with self.build_env: call_subprocess( [ - sys.executable, -+ os.environ.get('PIP_PYTHON_PATH', sys.executable), ++ sys_executable, '-c', SETUPTOOLS_SHIM % self.setup_py ] + -@@ -1015,7 +1015,7 @@ class InstallRequirement(object): +@@ -834,7 +836,8 @@ class InstallRequirement(object): def get_install_args(self, global_options, record_filename, root, prefix, pycompile): - install_args = [sys.executable, "-u"] -+ install_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable), "-u"] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ install_args = [sys_executable, "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py -index 2bc6b745..e552afc1 100644 +index b1983171..0bab231d 100644 --- a/pipenv/patched/pip/_internal/req/req_set.py +++ b/pipenv/patched/pip/_internal/req/req_set.py -@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) +@@ -12,13 +12,16 @@ logger = logging.getLogger(__name__) class RequirementSet(object): -- def __init__(self, require_hashes=False): -+ def __init__(self, require_hashes=False, ignore_compatibility=True): +- def __init__(self, require_hashes=False, check_supported_wheels=True): ++ def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): """Create a RequirementSet. """ -@@ -24,6 +24,7 @@ class RequirementSet(object): - self.unnamed_requirements = [] - self.successfully_downloaded = [] - self.reqs_to_cleanup = [] -+ self.ignore_compatibility = ignore_compatibility + self.requirements = OrderedDict() + self.require_hashes = require_hashes + self.check_supported_wheels = check_supported_wheels ++ if ignore_compatibility: ++ self.check_supported_wheels = False ++ self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False - def __str__(self): - reqs = [req for req in self.requirements.values() -@@ -65,7 +66,7 @@ class RequirementSet(object): - # environment markers. - if install_req.link and install_req.link.is_wheel: - wheel = Wheel(install_req.link.filename) -- if not wheel.supported(): -+ if not wheel.supported() and not self.ignore_compatibility: - raise InstallationError( - "%s is not a supported wheel on this platform." % - wheel.filename -@@ -151,7 +152,7 @@ class RequirementSet(object): + # Mapping of alias: real_name + self.requirement_aliases = {} +@@ -171,7 +174,7 @@ class RequirementSet(object): return self.requirements[name] if name in self.requirement_aliases: return self.requirements[self.requirement_aliases[name]] - raise KeyError("No project with the name %r" % project_name) -+ # raise KeyError("No project with the name %r" % project_name) ++ pass def cleanup_files(self): """Clean up files, remove builds.""" diff --git a/pipenv/patched/pip/_internal/resolve.py b/pipenv/patched/pip/_internal/resolve.py -index 8480e48c..ffc4aa7d 100644 +index 2d9f1c56..bedc2582 100644 --- a/pipenv/patched/pip/_internal/resolve.py +++ b/pipenv/patched/pip/_internal/resolve.py @@ -35,7 +35,7 @@ class Resolver(object): @@ -356,22 +353,12 @@ index 8480e48c..ffc4aa7d 100644 """Prepare a single requirements file. :return: A list of additional InstallRequirements to also install. -@@ -245,6 +249,9 @@ class Resolver(object): - # Tell user what we are doing for this requirement: - # obtain (editable), skipping, processing (local url), collecting - # (remote url or package name) -+ if ignore_requires_python or self.ignore_requires_python: -+ self.ignore_compatibility = True -+ - if req_to_install.constraint or req_to_install.prepared: - return [] - -@@ -260,11 +267,17 @@ class Resolver(object): +@@ -260,11 +264,17 @@ class Resolver(object): try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: - if self.ignore_requires_python: -+ if self.ignore_compatibility: ++ if self.ignore_requires_python or self.ignore_compatibility: logger.warning(err.args[0]) else: raise @@ -385,14 +372,13 @@ index 8480e48c..ffc4aa7d 100644 more_reqs = [] def add_req(subreq, extras_requested): -@@ -290,10 +303,14 @@ class Resolver(object): - # We add req_to_install before its dependencies, so that we +@@ -291,9 +301,13 @@ class Resolver(object): # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): + # 'unnamed' requirements will get added here + available_requested = sorted( + set(dist.extras) & set(req_to_install.extras) + ) - # 'unnamed' requirements will get added here req_to_install.is_direct = True requirement_set.add_requirement( req_to_install, parent_req_name=None, @@ -400,7 +386,7 @@ index 8480e48c..ffc4aa7d 100644 ) if not self.ignore_dependencies: -@@ -317,6 +334,19 @@ class Resolver(object): +@@ -317,6 +331,19 @@ class Resolver(object): for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) @@ -408,7 +394,7 @@ index 8480e48c..ffc4aa7d 100644 + for available in available_requested: + if hasattr(dist, '_DistInfoDistribution__dep_map'): + for req in dist._DistInfoDistribution__dep_map[available]: -+ req = InstallRequirement.from_req( ++ req = install_req_from_req( + str(req), + req_to_install, + isolated=self.isolated, @@ -420,24 +406,11 @@ index 8480e48c..ffc4aa7d 100644 if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took -diff --git a/pipenv/patched/pip/_internal/utils/misc.py b/pipenv/patched/pip/_internal/utils/misc.py -index 3236af63..439a831d 100644 ---- a/pipenv/patched/pip/_internal/utils/misc.py -+++ b/pipenv/patched/pip/_internal/utils/misc.py -@@ -96,7 +96,7 @@ def get_prog(): - try: - prog = os.path.basename(sys.argv[0]) - if prog in ('__main__.py', '-c'): -- return "%s -m pip" % sys.executable -+ return "%s -m pip" % os.environ.get('PIP_PYTHON_PATH', sys.executable) - else: - return prog - except (AttributeError, TypeError, IndexError): diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py -index 5f9bb93d..276a9ccc 100644 +index c43142f0..f241cce0 100644 --- a/pipenv/patched/pip/_internal/utils/packaging.py +++ b/pipenv/patched/pip/_internal/utils/packaging.py -@@ -28,7 +28,7 @@ def check_requires_python(requires_python): +@@ -29,7 +29,7 @@ def check_requires_python(requires_python): requires_python_specifier = specifiers.SpecifierSet(requires_python) # We only use major.minor.micro @@ -446,33 +419,21 @@ index 5f9bb93d..276a9ccc 100644 return python_version in requires_python_specifier -@@ -40,20 +40,17 @@ def get_metadata(dist): - return dist.get_metadata('PKG-INFO') +@@ -48,9 +48,11 @@ def get_metadata(dist): + return feed_parser.close() -def check_dist_requires_python(dist): +def check_dist_requires_python(dist, absorb=True): - metadata = get_metadata(dist) - feed_parser = FeedParser() - feed_parser.feed(metadata) - pkg_info_dict = feed_parser.close() + pkg_info_dict = get_metadata(dist) requires_python = pkg_info_dict.get('Requires-Python') -+ if not absorb: ++ if absorb: + return requires_python try: if not check_requires_python(requires_python): -- raise exceptions.UnsupportedPythonVersion( -- "%s requires Python '%s' but the running Python is %s" % ( -- dist.project_name, -- requires_python, -- '.'.join(map(str, sys.version_info[:3])),) -- ) -+ return requires_python - except specifiers.InvalidSpecifier as e: - logger.warning( - "Package %s has an invalid Requires-Python entry %s - %s", + raise exceptions.UnsupportedPythonVersion( diff --git a/pipenv/patched/pip/_internal/wheel.py b/pipenv/patched/pip/_internal/wheel.py -index fcf9d3d3..d8aff848 100644 +index 5ce890eb..46c0181c 100644 --- a/pipenv/patched/pip/_internal/wheel.py +++ b/pipenv/patched/pip/_internal/wheel.py @@ -83,7 +83,7 @@ def fix_script(path): @@ -484,28 +445,24 @@ index fcf9d3d3..d8aff848 100644 firstline = b'#!' + exename + os.linesep.encode("ascii") rest = script.read() with open(path, 'wb') as script: -@@ -665,7 +665,7 @@ class WheelBuilder(object): +@@ -167,7 +167,8 @@ def message_about_scripts_not_on_PATH(scripts): + ] + # If an executable sits with sys.executable, we don't warn for it. + # This covers the case of venv invocations without activating the venv. +- not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) ++ executable_loc = os.environ.get("PIP_PYTHON_PATH", sys.executable) ++ not_warn_dirs.append(os.path.normcase(os.path.dirname(executable_loc))) + warn_for = { + parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() + if os.path.normcase(parent_dir) not in not_warn_dirs +@@ -667,8 +668,9 @@ class WheelBuilder(object): + # isolating. Currently, it breaks Python in virtualenvs, because it # relies on site.py to find parts of the standard library outside the # virtualenv. ++ executable_loc = os.environ.get('PIP_PYTHON_PATH', sys.executable) return [ - sys.executable, '-u', '-c', -+ os.environ.get('PIP_PYTHON_PATH', sys.executable), '-u', '-c', ++ executable_loc, '-u', '-c', SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) -diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py -index 793dd1cb..426880e9 100644 ---- a/pipenv/patched/pip/_internal/index.py -+++ b/pipenv/patched/pip/_internal/index.py -@@ -477,7 +477,10 @@ class PackageFinder(object): - - page_versions = [] - for page in self._get_pages(url_locations, project_name): -- logger.debug('Analyzing links from page %s', page.url) -+ try: -+ logger.debug('Analyzing links from page %s', page.url) -+ except AttributeError: -+ continue - with indent_log(): - page_versions.extend( - self._package_versions(page.links, search) diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch index ae0bf088..7e42237f 100644 --- a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -1,8 +1,8 @@ -diff --git a/delegator.py b/delegator.py -index 25d21f0..582f4fe 100644 ---- a/delegator.py -+++ b/delegator.py -@@ -7,15 +7,18 @@ import locale +diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py +index d15aeb97..cf6f91c8 100644 +--- a/pipenv/vendor/delegator.py ++++ b/pipenv/vendor/delegator.py +@@ -7,6 +7,8 @@ import locale import errno from pexpect.popen_spawn import PopenSpawn @@ -11,96 +11,13 @@ index 25d21f0..582f4fe 100644 # Include `unicode` in STR_TYPES for Python 2.X try: - STR_TYPES = (str, unicode) - except NameError: -- STR_TYPES = (str, ) -+ STR_TYPES = (str,) - - TIMEOUT = 30 - -+ - def pid_exists(pid): - """Check whether pid exists in the current process table.""" - if pid == 0: -@@ -43,8 +46,8 @@ def pid_exists(pid): - else: - return True - --class Command(object): - -+class Command(object): - def __init__(self, cmd, timeout=TIMEOUT): - super(Command, self).__init__() - self.cmd = cmd -@@ -56,7 +59,7 @@ class Command(object): - self.__err = None - - def __repr__(self): -- return ''.format(self.cmd) -+ return "".format(self.cmd) - - @property - def _popen_args(self): -@@ -65,27 +68,23 @@ class Command(object): - @property - def _default_popen_kwargs(self): - return { -- 'env': os.environ.copy(), -- 'stdin': subprocess.PIPE, -- 'stdout': subprocess.PIPE, -- 'stderr': subprocess.PIPE, -- 'shell': True, -- 'universal_newlines': True, -- 'bufsize': 0 -+ "env": os.environ.copy(), -+ "stdin": subprocess.PIPE, -+ "stdout": subprocess.PIPE, -+ "stderr": subprocess.PIPE, -+ "shell": True, -+ "universal_newlines": True, -+ "bufsize": 0, - } - - @property - def _default_pexpect_kwargs(self): -- encoding = 'utf-8' -- if sys.platform == 'win32': -+ encoding = "utf-8" -+ if sys.platform == "win32": - default_encoding = locale.getdefaultlocale()[1] - if default_encoding is not None: - encoding = default_encoding -- return { -- 'env': os.environ.copy(), -- 'encoding': encoding, -- 'timeout': self.timeout -- } -+ return {"env": os.environ.copy(), "encoding": encoding, "timeout": self.timeout} - - @property - def _uses_subprocess(self): -@@ -99,18 +98,25 @@ class Command(object): - def std_out(self): - return self.subprocess.stdout - -+ @property -+ def ok(self): -+ return self.return_code == 0 -+ - @property - def _pexpect_out(self): - if self.subprocess.encoding: -- result = '' -+ result = "" - else: -- result = b'' -+ result = b"" - +@@ -110,8 +112,11 @@ class Command(object): if self.subprocess.before: result += self.subprocess.before - if self.subprocess.after: +- if self.subprocess.after: - result += self.subprocess.after ++ if self.subprocess.after and self.subprocess.after is not pexpect.EOF: + try: + result += self.subprocess.after + except (pexpect.EOF, pexpect.TIMEOUT): @@ -108,53 +25,17 @@ index 25d21f0..582f4fe 100644 result += self.subprocess.read() return result -@@ -148,7 +154,7 @@ class Command(object): - def pid(self): - """The process' PID.""" - # Support for pexpect's functionality. -- if hasattr(self.subprocess, 'proc'): -+ if hasattr(self.subprocess, "proc"): - return self.subprocess.proc.pid - # Standard subprocess method. - return self.subprocess.pid -@@ -177,23 +183,24 @@ class Command(object): +@@ -178,6 +183,7 @@ class Command(object): # Use subprocess. if self.blocking: popen_kwargs = self._default_popen_kwargs.copy() -- popen_kwargs['universal_newlines'] = not binary + del popen_kwargs["stdin"] -+ popen_kwargs["universal_newlines"] = not binary + popen_kwargs["universal_newlines"] = not binary if cwd: -- popen_kwargs['cwd'] = cwd -+ popen_kwargs["cwd"] = cwd - if env: -- popen_kwargs['env'].update(env) -+ popen_kwargs["env"].update(env) - s = subprocess.Popen(self._popen_args, **popen_kwargs) - # Otherwise, use pexpect. - else: - pexpect_kwargs = self._default_pexpect_kwargs.copy() - if binary: -- pexpect_kwargs['encoding'] = None -+ pexpect_kwargs["encoding"] = None - if cwd: -- pexpect_kwargs['cwd'] = cwd -+ pexpect_kwargs["cwd"] = cwd - if env: -- pexpect_kwargs['env'].update(env) -+ pexpect_kwargs["env"].update(env) - # Enable Python subprocesses to work with expect functionality. -- pexpect_kwargs['env']['PYTHONUNBUFFERED'] = '1' -+ pexpect_kwargs["env"]["PYTHONUNBUFFERED"] = "1" - s = PopenSpawn(self._popen_args, **pexpect_kwargs) - self.subprocess = s - self.was_run = True -@@ -202,15 +209,18 @@ class Command(object): - """Waits on the given pattern to appear in std_out""" - + popen_kwargs["cwd"] = cwd +@@ -205,7 +211,10 @@ class Command(object): if self.blocking: -- raise RuntimeError('expect can only be used on non-blocking commands.') -+ raise RuntimeError("expect can only be used on non-blocking commands.") + raise RuntimeError("expect can only be used on non-blocking commands.") - self.subprocess.expect(pattern=pattern, timeout=timeout) + try: @@ -164,14 +45,7 @@ index 25d21f0..582f4fe 100644 def send(self, s, end=os.linesep, signal=False): """Sends the given string or signal to std_in.""" - - if self.blocking: -- raise RuntimeError('send can only be used on non-blocking commands.') -+ raise RuntimeError("send can only be used on non-blocking commands.") - - if not signal: - if self._uses_subprocess: -@@ -233,14 +243,25 @@ class Command(object): +@@ -234,14 +243,25 @@ class Command(object): """Blocks until process is complete.""" if self._uses_subprocess: # consume stdout and stderr @@ -204,7 +78,7 @@ index 25d21f0..582f4fe 100644 def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next -@@ -262,7 +283,6 @@ class Command(object): +@@ -263,7 +283,6 @@ class Command(object): c.run(block=False, cwd=cwd) if data: c.send(data) @@ -212,24 +86,3 @@ index 25d21f0..582f4fe 100644 c.block() return c -@@ -273,12 +293,12 @@ def _expand_args(command): - # Prepare arguments. - if isinstance(command, STR_TYPES): - if sys.version_info[0] == 2: -- splitter = shlex.shlex(command.encode('utf-8')) -+ splitter = shlex.shlex(command.encode("utf-8")) - elif sys.version_info[0] == 3: - splitter = shlex.shlex(command) - else: -- splitter = shlex.shlex(command.encode('utf-8')) -- splitter.whitespace = '|' -+ splitter = shlex.shlex(command.encode("utf-8")) -+ splitter.whitespace = "|" - splitter.whitespace_split = True - command = [] - -@@ -319,4 +339,3 @@ def run(command, block=True, binary=False, timeout=TIMEOUT, cwd=None, env=None): - c.block() - - return c -- diff --git a/tasks/vendoring/patches/vendor/passa-close-session.patch b/tasks/vendoring/patches/vendor/passa-close-session.patch new file mode 100644 index 00000000..38846bef --- /dev/null +++ b/tasks/vendoring/patches/vendor/passa-close-session.patch @@ -0,0 +1,12 @@ +diff --git a/pipenv/vendor/passa/internals/dependencies.py b/pipenv/vendor/passa/internals/dependencies.py +index 53b19b17..358cc33b 100644 +--- a/pipenv/vendor/passa/internals/dependencies.py ++++ b/pipenv/vendor/passa/internals/dependencies.py +@@ -154,6 +154,7 @@ def _get_dependencies_from_json(ireq, sources): + return dependencies + except Exception as e: + print("unable to read dependencies via {0} ({1})".format(url, e)) ++ session.close() + return + + diff --git a/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch b/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch new file mode 100644 index 00000000..69e4cac7 --- /dev/null +++ b/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch @@ -0,0 +1,28 @@ +diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py +index 2a848922..57a90277 100644 +--- a/pipenv/vendor/vistir/spin.py ++++ b/pipenv/vendor/vistir/spin.py +@@ -5,6 +5,7 @@ import os + import signal + import sys + ++import colorama + import cursor + import six + +@@ -31,6 +32,7 @@ CLEAR_LINE = chr(27) + "[K" + + class DummySpinner(object): + def __init__(self, text="", **kwargs): ++ colorama.init() + self.text = to_native_string(text) + self.stdout = kwargs.get("stdout", sys.stdout) + self.stderr = kwargs.get("stderr", sys.stderr) +@@ -112,7 +114,6 @@ class VistirSpinner(base_obj): + """ + + self.handler = handler +- import colorama + colorama.init() + sigmap = {} + if handler: From 8a56a75cd544410fa39965aedcecf21539d812f0 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 27 Oct 2018 21:02:22 -0400 Subject: [PATCH 020/218] Fix virtualenv creation and error logging Signed-off-by: Dan Ryan --- pipenv/_compat.py | 15 ++++++++++-- pipenv/project.py | 25 ++++++++------------ pipenv/utils.py | 46 ++++++++++++++++++++++++++++++++++--- tasks/vendoring/__init__.py | 1 + 4 files changed, 66 insertions(+), 21 deletions(-) diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 087025a4..6cc4c732 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -11,6 +11,7 @@ import os import six import sys import warnings +import vistir from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp from .utils import logging, rmtree @@ -367,12 +368,22 @@ def force_encoding(): OUT_ENCODING, ERR_ENCODING = force_encoding() +UNICODE_TO_ASCII_TRANSLATION_MAP = { + 8230: u"...", + 8211: u"-" +} + + def decode_output(output): if not isinstance(output, six.string_types): return output try: output = output.encode(DEFAULT_ENCODING) - except AttributeError: - pass + except (AttributeError, UnicodeDecodeError): + if six.PY2: + output = unicode.translate(vistir.misc.to_text(output), + UNICODE_TO_ASCII_TRANSLATION_MAP) + else: + output = output.translate(UNICODE_TO_ASCII_TRANSLATION_MAP) output = output.decode(DEFAULT_ENCODING) return output diff --git a/pipenv/project.py b/pipenv/project.py index dcf9b417..f7e57245 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -15,6 +15,7 @@ import pipfile import pipfile.api import six import vistir +import virtualenv as _virtualenv import toml from .cmdparse import Script @@ -985,25 +986,17 @@ class Project(object): @property def env_paths(self): - import sysconfig location = self.virtualenv_location if self.virtualenv_location else sys.prefix - prefix = vistir.compat.Path(location).as_posix() - scheme = sysconfig._get_default_scheme() - if not scheme: - scheme = "posix_prefix" if not sys.platform == "win32" else "nt" - config = { - "base": prefix, - "installed_base": prefix, - "platbase": prefix, - "installed_platbase": prefix - } - config.update(self._pyversion) + prefix = vistir.compat.Path(location) + home, lib, inc, bin_ = _virtualenv.path_locations(prefix) paths = { - k: v.format(**config) - for k, v in sysconfig._INSTALL_SCHEMES[scheme].items() + "lib": lib, + "include": inc, + "scripts": bin_, + "purelib": lib, + "prefix": home, + "base": home } - if "prefix" not in paths: - paths["prefix"] = prefix return paths @cached_property diff --git a/pipenv/utils.py b/pipenv/utils.py index 893ed88f..9fa6e571 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -331,7 +331,7 @@ def venv_resolve_deps( pypi_mirror=None, ): from .vendor.vistir.misc import fs_str - from .vendor.vistir.compat import Path, to_native_string + from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError from .vendor.vistir.path import create_tracked_tempdir from .cmdparse import Script from .core import spinner @@ -370,7 +370,7 @@ def venv_resolve_deps( with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER, nospin=environments.PIPENV_NOSPIN) as sp: c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) - _out = to_native_string("") + _out = decode_output("") result = None while True: try: @@ -404,7 +404,7 @@ def venv_resolve_deps( try: return json.loads(c.out.split("RESULTS:")[1].strip()) - except IndexError: + except (IndexError, JSONDecodeError): click_echo(c.out.strip(), err=True) click_echo(c.err.strip(), err=True) raise RuntimeError("There was a problem with locking.") @@ -1305,3 +1305,43 @@ def parse_indexes(line): indexes = index + extra_indexes trusted_hosts = args.trusted_host if args.trusted_host else [] return indexes, trusted_hosts, remainder + + +def fix_venv_site(venv_lib_dir): + # From https://github.com/pypa/pip/blob/master/tests/lib/venv.py#L84 + # Prevent accidental inclusions of site packages during virtualenv operations + from .vendor.vistir.compat import Path + import compileall + site_py = Path(venv_lib_dir).joinpath('site.py').as_posix() + with open(site_py) as fp: + site_contents = fp.read() + for pattern, replace in ( + ( + # Ensure enabling user site does not result in adding + # the real site-packages' directory to `sys.path`. + ( + '\ndef virtual_addsitepackages(known_paths):\n' + ), + ( + '\ndef virtual_addsitepackages(known_paths):\n' + ' return known_paths\n' + ), + ), + ( + # Fix sites ordering: user site must be added before system. + ( + '\n paths_in_sys = addsitepackages(paths_in_sys)' + '\n paths_in_sys = addusersitepackages(paths_in_sys)\n' + ), + ( + '\n paths_in_sys = addusersitepackages(paths_in_sys)' + '\n paths_in_sys = addsitepackages(paths_in_sys)\n' + ), + ), + ): + if pattern in site_contents and replace not in site_contents: + site_contents = site_contents.replace(pattern, replace) + with open(site_py, 'w') as fp: + fp.write(site_contents) + # Make sure bytecode is up-to-date too. + assert compileall.compile_file(str(site_py), quiet=1, force=True) diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 4d968a7d..ea0d038c 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -52,6 +52,7 @@ HARDCODED_LICENSE_URLS = { 'distlib': 'https://github.com/vsajip/distlib/raw/master/LICENSE.txt', 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt', 'pyparsing': 'https://raw.githubusercontent.com/pyparsing/pyparsing/master/LICENSE', + 'resolvelib': 'https://raw.githubusercontent.com/sarugaku/resolvelib/master/LICENSE' } FILE_WHITE_LIST = ( From d069dff844c04babe4827eb33e443c180e83c5bb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 27 Oct 2018 21:46:11 -0400 Subject: [PATCH 021/218] Update project and spinner outputs Signed-off-by: Dan Ryan Try again Signed-off-by: Dan Ryan Fix test config to skip failed removals Signed-off-by: Dan Ryan Update piptools to handle some errors Signed-off-by: Dan Ryan Fix test config to skip failed removals Signed-off-by: Dan Ryan Update tempfile.py Use vistirs temporary directory implementation Update temp_dir.py Force pip to use weakrefs in tempdirs Fix pip implementation to set name of tempdir typo fix Signed-off-by: Dan Ryan fix pip tempdir implementation Signed-off-by: Dan Ryan Update tempfiles to use weakrefs Signed-off-by: Dan Ryan fix patch paths Signed-off-by: Dan Ryan Fix pip tempdir implementation Signed-off-by: Dan Ryan Syntax error fix Signed-off-by: Dan Ryan Unconstrain windows tests Signed-off-by: Dan Ryan Update dependencies, add news Signed-off-by: Dan Ryan Fix pythonfinder path search nesting bug - Fixes #3121 Signed-off-by: Dan Ryan Update requirementslib - Fix subdirectory issue Signed-off-by: Dan Ryan Fix logic error Signed-off-by: Dan Ryan conditional builds Signed-off-by: Dan Ryan --- .vsts-ci/linux.yml | 19 +++ .vsts-ci/steps/run-tests.yml | 2 +- .vsts-ci/windows.yml | 19 +++ news/3090.bugfix.rst | 1 + news/3094.bugfix.rst | 1 + news/3102.bugfix.rst | 1 + news/3109.bugfix.rst | 1 + news/3113.bugfix.rst | 1 + news/3114.bugfix.rst | 1 + news/3117.bugfix.rst | 1 + news/3121.bugfix.rst | 1 + news/3121.vendor.rst | 1 + pipenv/core.py | 3 +- .../notpip/_internal/utils/temp_dir.py | 38 ++++- pipenv/patched/piptools/repositories/pypi.py | 19 ++- pipenv/project.py | 33 +++- pipenv/resolver.py | 5 +- pipenv/utils.py | 26 ++- pipenv/vendor/pythonfinder/utils.py | 24 ++- pipenv/vendor/requirementslib/__init__.py | 2 +- .../requirementslib/models/requirements.py | 64 +++++--- pipenv/vendor/requirementslib/models/vcs.py | 26 ++- pipenv/vendor/resolvelib/LICENSE | 13 ++ pipenv/vendor/vendor.txt | 2 +- pipenv/vendor/vistir/__init__.py | 30 +++- pipenv/vendor/vistir/path.py | 65 +++++--- pipenv/vendor/vistir/spin.py | 2 +- tasks/vendoring/patches/patched/pip18.patch | 67 ++++++++ .../vendoring/patches/patched/piptools.patch | 152 +++++++++++------- tests/integration/conftest.py | 22 +-- 30 files changed, 478 insertions(+), 164 deletions(-) create mode 100644 news/3090.bugfix.rst create mode 100644 news/3094.bugfix.rst create mode 100644 news/3102.bugfix.rst create mode 100644 news/3109.bugfix.rst create mode 100644 news/3113.bugfix.rst create mode 100644 news/3114.bugfix.rst create mode 100644 news/3117.bugfix.rst create mode 100644 news/3121.bugfix.rst create mode 100644 news/3121.vendor.rst create mode 100644 pipenv/vendor/resolvelib/LICENSE diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 7968801c..95c62dc5 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -1,3 +1,22 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + phases: - template: phases/test.yml parameters: diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml index 4c2640ab..a7b99a13 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.vsts-ci/steps/run-tests.yml @@ -12,7 +12,7 @@ steps: $env:TEMP='T:\' Write-Host "##vso[task.setvariable variable=TMP]T:\" $env:TEMP='T:\' - D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor -k 'test_get_vcs_refs or test_install_editable_git_tag' --junitxml=test-results.xml tests + D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests - task: PublishTestResults@2 diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index a397a23c..e423c3ab 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -1,3 +1,22 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + phases: - template: phases/test.yml parameters: diff --git a/news/3090.bugfix.rst b/news/3090.bugfix.rst new file mode 100644 index 00000000..af772c3d --- /dev/null +++ b/news/3090.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug in ``requirementslib`` which prevented successful installation from mercurial repositories. diff --git a/news/3094.bugfix.rst b/news/3094.bugfix.rst new file mode 100644 index 00000000..e17b2e32 --- /dev/null +++ b/news/3094.bugfix.rst @@ -0,0 +1 @@ +Fixed random resource warnings when using pyenv or any other subprocess calls. diff --git a/news/3102.bugfix.rst b/news/3102.bugfix.rst new file mode 100644 index 00000000..2224ac0b --- /dev/null +++ b/news/3102.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3109.bugfix.rst b/news/3109.bugfix.rst new file mode 100644 index 00000000..af5718b6 --- /dev/null +++ b/news/3109.bugfix.rst @@ -0,0 +1 @@ +Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures. diff --git a/news/3113.bugfix.rst b/news/3113.bugfix.rst new file mode 100644 index 00000000..af43b87d --- /dev/null +++ b/news/3113.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue resolving virtualenv paths for users without ``platlib`` values on their systems. diff --git a/news/3114.bugfix.rst b/news/3114.bugfix.rst new file mode 100644 index 00000000..2224ac0b --- /dev/null +++ b/news/3114.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3117.bugfix.rst b/news/3117.bugfix.rst new file mode 100644 index 00000000..2224ac0b --- /dev/null +++ b/news/3117.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3121.bugfix.rst b/news/3121.bugfix.rst new file mode 100644 index 00000000..fb815c42 --- /dev/null +++ b/news/3121.bugfix.rst @@ -0,0 +1 @@ +Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. diff --git a/news/3121.vendor.rst b/news/3121.vendor.rst new file mode 100644 index 00000000..fb815c42 --- /dev/null +++ b/news/3121.vendor.rst @@ -0,0 +1 @@ +Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. diff --git a/pipenv/core.py b/pipenv/core.py index ee1658bf..59b9a29c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1355,7 +1355,8 @@ def pip_install( write_to_tmpfile = False if requirement: needs_hashes = not requirement.editable and not ignore_hashes and r is None - write_to_tmpfile = needs_hashes + has_subdir = requirement.is_vcs and requirement.req.subdirectory + write_to_tmpfile = needs_hashes or has_subdir if not trusted_hosts: trusted_hosts = [] diff --git a/pipenv/patched/notpip/_internal/utils/temp_dir.py b/pipenv/patched/notpip/_internal/utils/temp_dir.py index ba472b0d..893dc975 100644 --- a/pipenv/patched/notpip/_internal/utils/temp_dir.py +++ b/pipenv/patched/notpip/_internal/utils/temp_dir.py @@ -3,8 +3,10 @@ from __future__ import absolute_import import logging import os.path import tempfile +import warnings from pipenv.patched.notpip._internal.utils.misc import rmtree +from pipenv.vendor.vistir.compat import finalize, ResourceWarning logger = logging.getLogger(__name__) @@ -45,6 +47,20 @@ class TempDirectory(object): self.path = path self.delete = delete self.kind = kind + self._finalizer = None + if path: + self._register_finalizer() + + def _register_finalizer(self): + if self.delete and self.path: + self._finalizer = finalize( + self, + self._cleanup, + self.path, + warn_message=None + ) + else: + self._finalizer = None def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.path) @@ -72,11 +88,27 @@ class TempDirectory(object): self.path = os.path.realpath( tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) ) + self._register_finalizer() logger.debug("Created temporary directory: {}".format(self.path)) + @classmethod + def _cleanup(cls, name, warn_message=None): + try: + rmtree(name) + except OSError: + pass + else: + if warn_message: + warnings.warn(warn_message, ResourceWarning) + def cleanup(self): """Remove the temporary directory created and reset state """ - if self.path is not None and os.path.exists(self.path): - rmtree(self.path) - self.path = None + if getattr(self._finalizer, "detach", None) and self._finalizer.detach(): + if os.path.exists(self.path): + try: + rmtree(self.path) + except OSError: + pass + else: + self.path = None diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index c1871670..2a0743a3 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -19,11 +19,10 @@ from .._compat import ( InstallRequirement, SafeFileCache ) -os.environ["PIP_SHIMS_BASE_MODULE"] = str("notpip") +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") from pip_shims.shims import do_import, VcsSupport, WheelCache from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet, Specifier -from packaging.markers import Op, Value, Variable, Marker InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver @@ -31,7 +30,7 @@ from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR from ..exceptions import NoCandidateFound from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, - make_install_requirement, clean_requires_python) + make_install_requirement, clean_requires_python) from .base import BaseRepository try: @@ -243,6 +242,7 @@ class PyPIRepository(BaseRepository): dist = None ireq.isolated = False ireq._wheel_cache = wheel_cache + try: from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer except ImportError: @@ -295,7 +295,18 @@ class PyPIRepository(BaseRepository): resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() + + cleanup_fn = getattr(reqset, "cleanup_files", None) + if cleanup_fn is not None: + try: + cleanup_fn() + except OSError: + pass + + if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): + if ireq.editable: + self._source_dir = TemporaryDirectory(fs_str("source")) + ireq.ensure_has_source_dir(self.source_dir) if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # Collect setup_requires info from local eggs. diff --git a/pipenv/project.py b/pipenv/project.py index f7e57245..d25dba9a 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -15,7 +15,6 @@ import pipfile import pipfile.api import six import vistir -import virtualenv as _virtualenv import toml from .cmdparse import Script @@ -84,6 +83,8 @@ class _LockFileEncoder(json.JSONEncoder): if isinstance(obj, (ContainerElement, TokenElement)): return obj.primitive_value + elif isinstance(obj, vistir.compat.Path): + obj = obj.as_posix() return super(_LockFileEncoder, self).default(obj) def encode(self, obj): @@ -988,7 +989,35 @@ class Project(object): def env_paths(self): location = self.virtualenv_location if self.virtualenv_location else sys.prefix prefix = vistir.compat.Path(location) - home, lib, inc, bin_ = _virtualenv.path_locations(prefix) + import importlib + try: + _virtualenv = importlib.import_module("virtualenv") + except ImportError: + with vistir.contextmanagers.temp_path(): + from string import Formatter + formatter = Formatter() + import sysconfig + if getattr(sys, "real_prefix", None): + scheme = sysconfig._get_default_scheme() + sysconfig._INSTALL_SCHEMES["posix_prefix"]["purelib"] + if not scheme: + scheme = "posix_prefix" if not sys.platform == "win32" else "nt" + is_purelib = "purelib" in sysconfig._INSTALL_SCHEMES[scheme] + lib_key = "purelib" if is_purelib else "platlib" + lib = sysconfig._INSTALL_SCHEMES[scheme][lib_key] + fields = [field for _, field, _, _ in formatter.parse() if field] + config = { + "py_version_short": self._pyversion, + } + for field in fields: + if field not in config: + config[field] = prefix + sys.path = [ + os.path.join(sysconfig._INSTALL_SCHEMES[scheme][lib_key], "site-packages"), + ] + sys.path + six.reload_module(importlib) + _virtualenv = importlib.import_module("virtualenv") + home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix()) paths = { "lib": lib, "include": inc, diff --git a/pipenv/resolver.py b/pipenv/resolver.py index b5ea41d9..9ef46878 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -70,7 +70,6 @@ def _main(pre, clear, verbose, system, requirements_dir, packages): ) from pipenv.core import project - sources = ( replace_pypi_sources(project.pipfile_sources, pypi_mirror_source) if pypi_mirror_source @@ -111,8 +110,8 @@ def main(): parsed, remaining = parser.parse_known_args() # sys.argv = remaining parsed = handle_parsed_args(parsed) - _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.requirements_dir, - parsed.packages) + _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, + parsed.requirements_dir, parsed.packages) if __name__ == "__main__": diff --git a/pipenv/utils.py b/pipenv/utils.py index 9fa6e571..c0979863 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -345,7 +345,6 @@ def venv_resolve_deps( return [] req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") - cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() @@ -364,7 +363,6 @@ def venv_resolve_deps( os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") - out = to_native_string("") EOF.__module__ = "pexpect.exceptions" with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER, @@ -430,6 +428,8 @@ def resolve_deps( index_lookup = {} markers_lookup = {} python_path = which("python", allow_global=allow_global) + if not os.environ.get("PIP_SRC"): + os.environ["PIP_SRC"] = project.virtualenv_src_location backup_python_path = sys.executable results = [] if not deps: @@ -1116,23 +1116,19 @@ def get_vcs_deps( packages = getattr(project, section) except AttributeError: return [], [] - if os.environ.get("PIP_SRC"): - src_dir = Path( - os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src")) - ) - src_dir.mkdir(mode=0o775, exist_ok=True) - else: - src_dir = create_tracked_tempdir(prefix="pipenv-lock-dir") for pkg_name, pkg_pipfile in packages.items(): requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) name = requirement.normalized_name commit_hash = None if requirement.is_vcs: - with locked_repository(requirement) as repo: - commit_hash = repo.get_commit_hash() - lockfile[name] = requirement.pipfile_entry[1] - lockfile[name]['ref'] = commit_hash - reqs.append(requirement) + try: + with locked_repository(requirement) as repo: + commit_hash = repo.get_commit_hash() + lockfile[name] = requirement.pipfile_entry[1] + lockfile[name]['ref'] = commit_hash + reqs.append(requirement) + except OSError: + continue return reqs, lockfile @@ -1257,11 +1253,11 @@ def is_virtual_environment(path): @contextmanager def locked_repository(requirement): from .vendor.vistir.path import create_tracked_tempdir - src_dir = create_tracked_tempdir(prefix="pipenv-src") if not requirement.is_vcs: return original_base = os.environ.pop("PIP_SHIMS_BASE_MODULE", None) os.environ["PIP_SHIMS_BASE_MODULE"] = fs_str("pipenv.patched.notpip") + src_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-src") try: with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo: yield repo diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index fd5ac99d..ca07b42f 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function +import itertools import locale import os import subprocess @@ -21,6 +22,9 @@ try: except ImportError: from backports.functools_lru_cache import lru_cache +six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) +from six.moves import Iterable + PYTHON_IMPLEMENTATIONS = ( "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", @@ -123,7 +127,21 @@ def filter_pythons(path): return filter(lambda x: path_is_python(x), path.iterdir()) +# def unnest(item): +# if isinstance(next((i for i in item), None), (list, tuple)): +# return chain(*filter(None, item)) +# return chain(filter(None, item)) + + def unnest(item): - if isinstance(next((i for i in item), None), (list, tuple)): - return chain(*filter(None, item)) - return chain(filter(None, item)) + if isinstance(item, Iterable) and not isinstance(item, six.string_types): + item, target = itertools.tee(item, 2) + else: + target = item + for el in target: + if isinstance(el, Iterable) and not isinstance(el, six.string_types): + el, el_copy = itertools.tee(el, 2) + for sub in unnest(el_copy): + yield sub + else: + yield el diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index ba40e5f5..8ceccd79 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.2.1' +__version__ = '1.2.2' import logging diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 3a029cbc..b3bc0132 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -16,7 +16,7 @@ from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement from packaging.specifiers import Specifier, SpecifierSet from packaging.utils import canonicalize_name -from pip_shims.shims import _strip_extras, parse_version, path_to_url, url_to_path +from pip_shims.shims import _strip_extras, parse_version, path_to_url, url_to_path, Link from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote from vistir.compat import FileNotFoundError, Path @@ -41,11 +41,11 @@ from .utils import ( ) -@attr.s +@attr.s(slots=True) class NamedRequirement(BaseRequirement): name = attr.ib() version = attr.ib(validator=attr.validators.optional(validate_specifiers)) - req = attr.ib() + req = attr.ib(type=PkgResourcesRequirement) extras = attr.ib(default=attr.Factory(list)) editable = attr.ib(default=False) @@ -109,21 +109,28 @@ LinkInfo = collections.namedtuple( ) -@attr.s +@attr.s(slots=True) class FileRequirement(BaseRequirement): """File requirements for tar.gz installable files or wheels or setup.py containing directories.""" + #: Path to the relevant `setup.py` location setup_path = attr.ib(default=None) + #: path to hit - without any of the VCS prefixes (like git+ / http+ / etc) path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) - # : path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - editable = attr.ib(default=False, type=bool) - extras = attr.ib(default=attr.Factory(list), type=list) - uri = attr.ib(type=six.string_types) + #: Whether the package is editable + editable = attr.ib(default=False) + #: Extras if applicable + extras = attr.ib(default=attr.Factory(list)) + #: URI of the package + uri = attr.ib() + #: Link object representing the package to clone link = attr.ib() - name = attr.ib(type=six.string_types) - req = attr.ib(type=PkgResourcesRequirement) - _has_hashed_name = False + _has_hashed_name = attr.ib(default=False) + #: Package name + name = attr.ib() + #: A :class:`~pkg_resources.Requirement` isntance + req = attr.ib() _uri_scheme = attr.ib(default=None) @classmethod @@ -470,14 +477,19 @@ class FileRequirement(BaseRequirement): return {name: pipfile_dict} -@attr.s +@attr.s(slots=True) class VCSRequirement(FileRequirement): + #: Whether the repository is editable editable = attr.ib(default=None) + #: URI for the repository uri = attr.ib(default=None) + #: path to the repository, if it's local path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) + #: vcs type, i.e. git/hg/svn vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) - # : vcs reference name (branch / commit / tag) + #: vcs reference name (branch / commit / tag) ref = attr.ib(default=None) + #: Subdirectory to use for installation if applicable subdirectory = attr.ib(default=None) _repo = attr.ib(default=None) _base_line = attr.ib(default=None) @@ -585,16 +597,28 @@ class VCSRequirement(FileRequirement): def get_vcs_repo(self, src_dir=None): from .vcs import VCSRepository checkout_dir = self.get_checkout_dir(src_dir=src_dir) - url = "{0}#egg={1}".format(self.vcs_uri, self.name) + url = build_vcs_link( + self.vcs, + self.uri, + name=self.name, + ref=self.ref, + subdirectory=self.subdirectory, + extras=self.extras + ) vcsrepo = VCSRepository( url=url, name=self.name, ref=self.ref if self.ref else None, checkout_directory=checkout_dir, - vcs_type=self.vcs + vcs_type=self.vcs, + subdirectory=self.subdirectory ) if not self.is_local: vcsrepo.obtain() + if self.subdirectory: + self.setup_path = os.path.join(checkout_dir, self.subdirectory, "setup.py") + else: + self.setup_path = os.path.join(checkout_dir, "setup.py") return vcsrepo def get_commit_hash(self): @@ -612,15 +636,15 @@ class VCSRequirement(FileRequirement): if not self.is_local and ref is not None: self.repo.checkout_ref(ref) repo_hash = self.repo.get_commit_hash() + self.req.revision = repo_hash return repo_hash @contextmanager def locked_vcs_repo(self, src_dir=None): + if not src_dir: + src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") vcsrepo = self.get_vcs_repo(src_dir=src_dir) - if self.ref and not self.is_local: - vcsrepo.checkout_ref(self.ref) - self.ref = self.get_commit_hash() - self.req.revision = self.ref + self.req.revision = vcsrepo.get_commit_hash() # Remove potential ref in the end of uri after ref is parsed if "@" in self.link.show_url and "@" in self.uri: @@ -1070,7 +1094,7 @@ class Requirement(object): if self.editable or self.req.editable: if ireq_line.startswith("-e "): ireq_line = ireq_line[len("-e "):] - with ensure_setup_py(self.req.path): + with ensure_setup_py(self.req.setup_path): ireq = ireq_from_editable(ireq_line) else: ireq = ireq_from_line(ireq_line) diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index fb2e6bc3..4efb9bd3 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -1,7 +1,6 @@ # -*- coding=utf-8 -*- import attr from pip_shims import VcsSupport, parse_version, pip_version -import vistir import os @@ -14,6 +13,7 @@ class VCSRepository(object): name = attr.ib() checkout_directory = attr.ib() vcs_type = attr.ib() + subdirectory = attr.ib(default=None) commit_sha = attr.ib(default=None) ref = attr.ib(default=None) repo_instance = attr.ib() @@ -31,35 +31,31 @@ class VCSRepository(object): return url.startswith("file") def obtain(self): - if not os.path.exists(self.checkout_directory): + if (os.path.exists(self.checkout_directory) and not + self.repo_instance.is_repository_directory(self.checkout_directory)): + self.repo_instance.unpack(self.checkout_directory) + elif not os.path.exists(self.checkout_directory): self.repo_instance.obtain(self.checkout_directory) - if self.ref: - self.checkout_ref(self.ref) - self.commit_sha = self.get_commit_hash(self.ref) else: - if not self.commit_sha: - self.commit_sha = self.get_commit_hash() + if self.ref: + self.checkout_ref(self.ref) + if not self.commit_sha: + self.commit_sha = self.get_commit_hash() def checkout_ref(self, ref): if not self.repo_instance.is_commit_id_equal( - self.checkout_directory, self.get_commit_hash(ref) + self.checkout_directory, self.get_commit_hash() ) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref): if not self.is_local: self.update(ref) def update(self, ref): target_ref = self.repo_instance.make_rev_options(ref) - sha = self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) - target_rev = target_ref.make_new(sha) if parse_version(pip_version) > parse_version("18.0"): self.repo_instance.update(self.checkout_directory, self.url, target_ref) else: self.repo_instance.update(self.checkout_directory, target_ref) - self.commit_hash = self.get_commit_hash(ref) + self.commit_sha = self.get_commit_hash() def get_commit_hash(self, ref=None): - if ref: - target_ref = self.repo_instance.make_rev_options(ref) - return self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) - # return self.repo_instance.get_revision(self.checkout_directory) return self.repo_instance.get_revision(self.checkout_directory) diff --git a/pipenv/vendor/resolvelib/LICENSE b/pipenv/vendor/resolvelib/LICENSE new file mode 100644 index 00000000..b9077766 --- /dev/null +++ b/pipenv/vendor/resolvelib/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2018, Tzu-ping Chung + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 35a32eb6..18352c2e 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -27,7 +27,7 @@ requests==2.20.0 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.2.1 +requirementslib==1.2.2 attrs==18.2.0 distlib==0.2.8 packaging==18.0 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index 912ab0a4..c8a995fa 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -1,7 +1,12 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals -from .compat import NamedTemporaryFile, TemporaryDirectory, partialmethod, to_native_string +from .compat import ( + NamedTemporaryFile, + TemporaryDirectory, + partialmethod, + to_native_string, +) from .contextmanagers import ( atomic_open_for_write, cd, @@ -10,12 +15,23 @@ from .contextmanagers import ( temp_path, spinner, ) -from .misc import load_path, partialclass, run, shell_escape +from .misc import ( + load_path, + partialclass, + run, + shell_escape, + decode_for_output, + to_text, + to_bytes, + take, + chunked, + divide, +) from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile from .spin import VistirSpinner, create_spinner -__version__ = '0.2.2' +__version__ = "0.2.3" __all__ = [ @@ -38,5 +54,11 @@ __all__ = [ "create_spinner", "create_tracked_tempdir", "create_tracked_tempfile", - "to_native_string" + "to_native_string", + "decode_for_output", + "to_text", + "to_bytes", + "take", + "chunked", + "divide", ] diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 68e6d464..ba008159 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -8,6 +8,7 @@ import os import posixpath import shutil import stat +import sys import warnings import six @@ -166,11 +167,12 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ - from .misc import to_bytes + from .compat import to_native_string - fn = to_bytes(fn, encoding="utf-8") + fn = to_native_string(fn) if os.path.exists(fn): - return bool(os.stat(fn).st_mode & stat.S_IREAD) and not os.access(fn, os.W_OK) + file_stat = os.stat(fn).st_mode + return not bool(file_stat & stat.S_IWRITE) or not os.access(fn, os.W_OK) return False @@ -182,9 +184,10 @@ def mkdir_p(newdir, mode=0o777): :raises: OSError if a file is encountered along the way """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ - from .misc import to_bytes, to_text + from .misc import to_text + from .compat import to_native_string - newdir = to_bytes(newdir, "utf-8") + newdir = to_native_string(newdir) if os.path.exists(newdir): if not os.path.isdir(newdir): raise OSError( @@ -193,9 +196,9 @@ def mkdir_p(newdir, mode=0o777): ) ) else: - head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) + head, tail = os.path.split(newdir) # Make sure the tail doesn't point to the asame place as the head - curdir = to_bytes(".", encoding="utf-8") + curdir = to_native_string(".") tail_and_head_match = ( os.path.relpath(tail, start=os.path.basename(head)) == curdir ) @@ -242,7 +245,7 @@ def create_tracked_tempdir(*args, **kwargs): tempdir = TemporaryDirectory(*args, **kwargs) TRACKED_TEMPORARY_DIRECTORIES.append(tempdir) atexit.register(tempdir.cleanup) - warnings.simplefilter("default", ResourceWarning) + warnings.simplefilter("ignore", ResourceWarning) return tempdir.name @@ -266,12 +269,20 @@ def set_write_bit(fn): :param str fn: The target filename or path """ - from .misc import to_bytes, locale_encoding + from .compat import to_native_string - fn = to_bytes(fn, encoding=locale_encoding) + fn = to_native_string(fn) if not os.path.exists(fn): return - os.chmod(fn, stat.S_IWRITE | stat.S_IWUSR | stat.S_IRUSR) + file_stat = os.stat(fn).st_mode + os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + if not os.path.isdir(fn): + return + for root, dirs, files in os.walk(fn, topdown=False): + for dir_ in [os.path.join(root,d) for d in dirs]: + set_write_bit(dir_) + for file_ in [os.path.join(root, f) for f in files]: + set_write_bit(file_) def rmtree(directory, ignore_errors=False): @@ -288,14 +299,14 @@ def rmtree(directory, ignore_errors=False): Setting `ignore_errors=True` may cause this to silently fail to delete the path """ - from .misc import locale_encoding, to_bytes + from .compat import to_native_string - directory = to_bytes(directory, encoding=locale_encoding) + directory = to_native_string(directory) try: shutil.rmtree( directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly ) - except (IOError, OSError) as exc: + except (IOError, OSError, FileNotFoundError) as exc: # Ignore removal failures where the file doesn't exist if exc.errno == errno.ENOENT: pass @@ -316,23 +327,24 @@ def handle_remove_readonly(func, path, exc): :func:`set_write_bit` on the target path and try again. """ # Check for read-only attribute - from .compat import ResourceWarning - from .misc import to_bytes + from .compat import ResourceWarning, FileNotFoundError, to_native_string - PERM_ERRORS = (errno.EACCES, errno.EPERM) + PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) default_warning_message = ( "Unable to remove file due to permissions restriction: {!r}" ) # split the initial exception out into its type, exception, and traceback exc_type, exc_exception, exc_tb = exc - path = to_bytes(path, encoding="utf-8") + path = to_native_string(path) if is_readonly_path(path): # Apply write permission and call original function set_write_bit(path) try: func(path) - except (OSError, IOError) as e: - if e.errno in PERM_ERRORS: + except (OSError, IOError, FileNotFoundError) as e: + if e.errno == errno.ENOENT: + return + elif e.errno in PERM_ERRORS: warnings.warn(default_warning_message.format(path), ResourceWarning) return @@ -340,17 +352,20 @@ def handle_remove_readonly(func, path, exc): set_write_bit(path) try: func(path) - except (OSError, IOError) as e: + except (OSError, IOError, FileNotFoundError) as e: if e.errno in PERM_ERRORS: warnings.warn(default_warning_message.format(path), ResourceWarning) + pass elif e.errno == errno.ENOENT: # File already gone - return + pass else: raise - return else: - raise - raise exc + return + elif exc_exception.errno == errno.ENOENT: + pass + else: + raise exc_exception def walk_up(bottom): diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index 6b6e498f..20587d9d 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -48,7 +48,7 @@ class DummySpinner(object): if exc_type: import traceback from .misc import decode_for_output - self.write_err(decode_for_output(traceback.format_exception(traceback))) + self.write_err(decode_for_output(traceback.format_exception(*sys.exc_info()))) self._close_output_buffer() return False diff --git a/tasks/vendoring/patches/patched/pip18.patch b/tasks/vendoring/patches/patched/pip18.patch index 150ee32f..f4e607c1 100644 --- a/tasks/vendoring/patches/patched/pip18.patch +++ b/tasks/vendoring/patches/patched/pip18.patch @@ -19,6 +19,73 @@ index 96f3b65c..cc5b3d15 100644 "python": platform.python_version(), "implementation": { "name": platform.python_implementation(), +diff --git a/pipenv/patched/pip/_internal/utils/temp_dir.py b/pipenv/patched/pip/_internal/utils/temp_dir.py +index edc506bf..84d57dac 100644 +--- a/pipenv/patched/pip/_internal/utils/temp_dir.py ++++ b/pipenv/patched/pip/_internal/utils/temp_dir.py +@@ -3,8 +3,10 @@ from __future__ import absolute_import + import logging + import os.path + import tempfile ++import warnings + + from pip._internal.utils.misc import rmtree ++from pipenv.vendor.vistir.compat import finalize, ResourceWarning + + logger = logging.getLogger(__name__) + +@@ -45,6 +47,20 @@ class TempDirectory(object): + self.path = path + self.delete = delete + self.kind = kind ++ self._finalizer = None ++ if path: ++ self._register_finalizer() ++ ++ def _register_finalizer(self): ++ if self.delete and self.path: ++ self._finalizer = finalize( ++ self, ++ self._cleanup, ++ self.path, ++ warn_message=None ++ ) ++ else: ++ self._finalizer = None + + def __repr__(self): + return "<{} {!r}>".format(self.__class__.__name__, self.path) +@@ -72,11 +88,27 @@ class TempDirectory(object): + self.path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) + ) ++ self._register_finalizer() + logger.debug("Created temporary directory: {}".format(self.path)) + ++ @classmethod ++ def _cleanup(cls, name, warn_message=None): ++ try: ++ rmtree(name) ++ except OSError: ++ pass ++ else: ++ if warn_message: ++ warnings.warn(warn_message, ResourceWarning) ++ + def cleanup(self): + """Remove the temporary directory created and reset state + """ +- if self.path is not None and os.path.exists(self.path): +- rmtree(self.path) +- self.path = None ++ if getattr(self._finalizer, "detach", None) and self._finalizer.detach(): ++ if os.path.exists(self.path): ++ try: ++ rmtree(self.path) ++ except OSError: ++ pass ++ else: ++ self.path = None diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py index 8c2f24f1..cdd48874 100644 --- a/pipenv/patched/pip/_internal/index.py diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 1db5ef44..3799ccf4 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -12,30 +12,56 @@ index 1fa3805..c0ecec8 100644 install_req_from_editable, ) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py -index 28da51f..de9b435 100644 +index 28da51f..c466ef0 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py -@@ -1,12 +1,13 @@ +@@ -1,45 +1,55 @@ # -*- coding=utf-8 -*- - import importlib +-import importlib -import pip -+from pip_shims import pip_version - import pkg_resources +-import pkg_resources ++__all__ = [ ++ "InstallRequirement", ++ "parse_requirements", ++ "RequirementSet", ++ "user_cache_dir", ++ "FAVORITE_HASH", ++ "is_file_url", ++ "url_to_path", ++ "PackageFinder", ++ "FormatControl", ++ "Wheel", ++ "Command", ++ "cmdoptions", ++ "get_installed_distributions", ++ "PyPI", ++ "SafeFileCache", ++ "InstallationError", ++ "parse_version", ++ "pip_version", ++ "install_req_from_editable", ++ "install_req_from_line", ++ "user_cache_dir" ++] -def do_import(module_path, subimport=None, old_path=None): -+def do_import(module_path, subimport=None, old_path=None, vendored_name=None): - old_path = old_path or module_path +- old_path = old_path or module_path - prefixes = ["pip._internal", "pip"] -+ prefix = vendored_name if vendored_name else "pip" -+ prefixes = ["{0}._internal".format(prefix), "{0}".format(prefix)] - paths = [module_path, old_path] - search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] - package = subimport if subimport else None -@@ -21,25 +22,28 @@ def do_import(module_path, subimport=None, old_path=None): - return getattr(imported, package) - - +- paths = [module_path, old_path] +- search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] +- package = subimport if subimport else None +- for to_import in search_order: +- if not subimport: +- to_import, _, package = to_import.rpartition(".") +- try: +- imported = importlib.import_module(to_import) +- except ImportError: +- continue +- else: +- return getattr(imported, package) +- +- -InstallRequirement = do_import('req.req_install', 'InstallRequirement') -parse_requirements = do_import('req.req_file', 'parse_requirements') -RequirementSet = do_import('req.req_set', 'RequirementSet') @@ -50,34 +76,38 @@ index 28da51f..de9b435 100644 -cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions') -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') -PyPI = do_import('models.index', 'PyPI') -+InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name="notpip") -+parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name="notpip") -+RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name="notpip") -+user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name="notpip") -+FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name="notpip") -+is_file_url = do_import('download', 'is_file_url', vendored_name="notpip") -+url_to_path = do_import('download', 'url_to_path', vendored_name="notpip") -+PackageFinder = do_import('index', 'PackageFinder', vendored_name="notpip") -+FormatControl = do_import('index', 'FormatControl', vendored_name="notpip") -+Wheel = do_import('wheel', 'Wheel', vendored_name="notpip") -+Command = do_import('cli.base_command', 'Command', old_path='basecommand', vendored_name="notpip") -+cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions', vendored_name="notpip") -+get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name="notpip") -+PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') -+SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') -+InstallationError = do_import('exceptions', 'InstallationError', vendored_name='notpip') ++from pipenv.vendor.appdirs import user_cache_dir ++from pip_shims.shims import ( ++ InstallRequirement, ++ parse_requirements, ++ RequirementSet, ++ FAVORITE_HASH, ++ is_file_url, ++ url_to_path, ++ PackageFinder, ++ FormatControl, ++ Wheel, ++ Command, ++ cmdoptions, ++ get_installed_distributions, ++ PyPI, ++ SafeFileCache, ++ InstallationError, ++ parse_version, ++ pip_version, ++) # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('18.1'): -+if pkg_resources.parse_version(pip_version) < pkg_resources.parse_version('18.1'): ++if parse_version(pip_version) < parse_version('18.1'): install_req_from_line = InstallRequirement.from_line install_req_from_editable = InstallRequirement.from_editable else: - install_req_from_line = do_import('req.constructors', 'install_req_from_line') - install_req_from_editable = do_import('req.constructors', 'install_req_from_editable') -+ install_req_from_line = do_import('req.constructors', 'install_req_from_line', vendored_name="notpip") -+ install_req_from_editable = do_import('req.constructors', 'install_req_from_editable', vendored_name="notpip") -+ ++ from pip_shims.shims import ( ++ install_req_from_editable, install_req_from_line ++ ) diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py index 08dabe1..480ad1e 100644 --- a/pipenv/patched/piptools/repositories/local.py @@ -92,7 +122,7 @@ index 08dabe1..480ad1e 100644 else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index bf69803..a1a3906 100644 +index bf69803..31b85b9 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,7 +1,7 @@ @@ -104,7 +134,7 @@ index bf69803..a1a3906 100644 import hashlib import os from contextlib import contextmanager -@@ -15,13 +15,23 @@ from .._compat import ( +@@ -15,13 +15,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -113,11 +143,10 @@ index bf69803..a1a3906 100644 + InstallRequirement, + SafeFileCache ) -+os.environ["PIP_SHIMS_BASE_MODULE"] = str("notpip") ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") +from pip_shims.shims import do_import, VcsSupport, WheelCache +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet, Specifier -+from packaging.markers import Op, Value, Variable, Marker +InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) +from pip._internal.resolve import Resolver as PipResolver + @@ -128,11 +157,11 @@ index bf69803..a1a3906 100644 -from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, -+ make_install_requirement, clean_requires_python) ++ make_install_requirement, clean_requires_python) from .base import BaseRepository try: -@@ -31,10 +41,44 @@ except ImportError: +@@ -31,10 +40,44 @@ except ImportError: def RequirementTracker(): yield @@ -181,7 +210,7 @@ index bf69803..a1a3906 100644 class PyPIRepository(BaseRepository): -@@ -46,8 +90,9 @@ class PyPIRepository(BaseRepository): +@@ -46,8 +89,9 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -192,7 +221,7 @@ index bf69803..a1a3906 100644 self.pip_options = pip_options index_urls = [pip_options.index_url] + pip_options.extra_index_urls -@@ -73,6 +118,10 @@ class PyPIRepository(BaseRepository): +@@ -73,6 +117,10 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -203,7 +232,7 @@ index bf69803..a1a3906 100644 # Setup file paths self.freshen_build_caches() -@@ -113,10 +162,13 @@ class PyPIRepository(BaseRepository): +@@ -113,10 +161,13 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -219,7 +248,7 @@ index bf69803..a1a3906 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,25 +178,86 @@ class PyPIRepository(BaseRepository): +@@ -126,25 +177,87 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( @@ -236,8 +265,7 @@ index bf69803..a1a3906 100644 + def gen(ireq): + if self.DEFAULT_INDEX_URL not in self.finder.index_urls: + return - -- def resolve_reqs(self, download_dir, ireq, wheel_cache): ++ + url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) + releases = self.session.get(url).json()['releases'] + @@ -266,7 +294,8 @@ index bf69803..a1a3906 100644 + try: + if ireq not in self._json_dep_cache: + self._json_dep_cache[ireq] = [g for g in gen(ireq)] -+ + +- def resolve_reqs(self, download_dir, ireq, wheel_cache): + return set(self._json_dep_cache[ireq]) + except Exception: + return set() @@ -291,6 +320,7 @@ index bf69803..a1a3906 100644 + dist = None + ireq.isolated = False + ireq._wheel_cache = wheel_cache ++ try: from pip._internal.operations.prepare import RequirementPreparer - from pip._internal.resolve import Resolver as PipResolver @@ -330,7 +360,7 @@ index bf69803..a1a3906 100644 } resolver = None preparer = None -@@ -177,15 +291,98 @@ class PyPIRepository(BaseRepository): +@@ -177,15 +291,109 @@ class PyPIRepository(BaseRepository): resolver_kwargs['preparer'] = preparer reqset = RequirementSet() ireq.is_direct = True @@ -339,9 +369,21 @@ index bf69803..a1a3906 100644 resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() +- reqset.cleanup_files() - return set(results) ++ cleanup_fn = getattr(reqset, "cleanup_files", None) ++ if cleanup_fn is not None: ++ try: ++ cleanup_fn() ++ except OSError: ++ pass ++ ++ if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): ++ if ireq.editable: ++ self._source_dir = TemporaryDirectory(fs_str("source")) ++ ireq.ensure_has_source_dir(self.source_dir) ++ + if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): + # Collect setup_requires info from local eggs. + # Do this after we call the preparer on these reqs to make sure their @@ -432,7 +474,7 @@ index bf69803..a1a3906 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -200,6 +397,7 @@ class PyPIRepository(BaseRepository): +@@ -200,6 +408,7 @@ class PyPIRepository(BaseRepository): # If a download_dir is passed, pip will unnecessarely # archive the entire source directory download_dir = None @@ -440,7 +482,7 @@ index bf69803..a1a3906 100644 elif ireq.link and not ireq.link.is_artifact: # No download_dir for VCS sources. This also works around pip # using git-checkout-index, which gets rid of the .git dir. -@@ -214,7 +412,8 @@ class PyPIRepository(BaseRepository): +@@ -214,7 +423,8 @@ class PyPIRepository(BaseRepository): wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) prev_tracker = os.environ.get('PIP_REQ_TRACKER') try: @@ -450,7 +492,7 @@ index bf69803..a1a3906 100644 finally: if 'PIP_REQ_TRACKER' in os.environ: if prev_tracker: -@@ -236,6 +435,10 @@ class PyPIRepository(BaseRepository): +@@ -236,6 +446,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -461,7 +503,7 @@ index bf69803..a1a3906 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -243,24 +446,22 @@ class PyPIRepository(BaseRepository): +@@ -243,24 +457,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 5f5c1930..0ab0ab22 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -56,9 +56,6 @@ def check_github_ssh(): return res -WE_HAVE_INTERNET = check_internet() -WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() - TESTS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PYPI_VENDOR_DIR = os.path.join(TESTS_ROOT, 'pypi') prepare_pypi_packages(PYPI_VENDOR_DIR) @@ -71,10 +68,13 @@ def pytest_runtest_setup(item): pytest.skip('requires github ssh') -@pytest.yield_fixture +@pytest.fixture def pathlib_tmpdir(request, tmpdir): yield Path(str(tmpdir)) - tmpdir.remove(ignore_errors=True) + try: + tmpdir.remove(ignore_errors=True) + except Exception: + pass # Borrowed from pip's test runner filesystem isolation @@ -102,6 +102,10 @@ def isolate(pathlib_tmpdir): os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) +WE_HAVE_INTERNET = check_internet() +WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() + + class _PipenvInstance(object): """An instance of a Pipenv Project...""" def __init__(self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None): @@ -207,14 +211,12 @@ def PipenvInstance(): yield _PipenvInstance -@pytest.fixture(scope='module') -def pip_src_dir(request): +@pytest.fixture(autouse=True) +def pip_src_dir(request, pathlib_tmpdir): old_src_dir = os.environ.get('PIP_SRC', '') - new_src_dir = TemporaryDirectory(prefix='pipenv-', suffix='-testsrc') - os.environ['PIP_SRC'] = fs_str(new_src_dir.name) + os.environ['PIP_SRC'] = pathlib_tmpdir.as_posix() def finalize(): - new_src_dir.cleanup() os.environ['PIP_SRC'] = fs_str(old_src_dir) request.addfinalizer(finalize) From 8459cd8d45de720e383faebb80de2f998bde0722 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 30 Oct 2018 01:11:54 -0400 Subject: [PATCH 022/218] Fix requirementslib update Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/requirements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index b3bc0132..4f392696 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -597,7 +597,7 @@ class VCSRequirement(FileRequirement): def get_vcs_repo(self, src_dir=None): from .vcs import VCSRepository checkout_dir = self.get_checkout_dir(src_dir=src_dir) - url = build_vcs_link( + link = build_vcs_link( self.vcs, self.uri, name=self.name, @@ -606,7 +606,7 @@ class VCSRequirement(FileRequirement): extras=self.extras ) vcsrepo = VCSRepository( - url=url, + url=link.url, name=self.name, ref=self.ref if self.ref else None, checkout_directory=checkout_dir, From 2cbf0e050c008127d19e3e06ff5e6ffa499a2f66 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 30 Oct 2018 01:58:15 -0400 Subject: [PATCH 023/218] Exclude setup_path from pipfile Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 4f392696..08eb5618 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -753,7 +753,7 @@ class VCSRequirement(FileRequirement): @property def pipfile_part(self): - excludes = ["_repo", "_base_line"] + excludes = ["_repo", "_base_line", "setup_path"] filter_func = lambda k, v: bool(v) is True and k.name not in excludes pipfile_dict = attr.asdict(self, filter=filter_func).copy() if "vcs" in pipfile_dict: From e8231dc756aa85ba019a23fce035032c56e38c2a Mon Sep 17 00:00:00 2001 From: jr7square Date: Thu, 4 Oct 2018 20:40:45 -0400 Subject: [PATCH 024/218] Provide a good message for error described in issue #2846 --- pipenv/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pipenv/core.py b/pipenv/core.py index 59b9a29c..e1243d79 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1775,6 +1775,9 @@ def do_install( # Don't search for requirements.txt files if the user provides one if requirements or package_args or project.pipfile_exists: skip_requirements = True + # Don't attempt to install develop and default packages if Pipfile is missing + if not project.pipfile_exists and not packages and dev: + click.echo("Could not find Pipfile.", err=True) concurrent = not sequential # Ensure that virtualenv is available. ensure_project( From 5caf9ac367b326aa631241e979759fddd061750d Mon Sep 17 00:00:00 2001 From: jr7square Date: Fri, 5 Oct 2018 20:16:23 -0400 Subject: [PATCH 025/218] aborting execution --- pipenv/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipenv/core.py b/pipenv/core.py index e1243d79..b23534d6 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1778,6 +1778,7 @@ def do_install( # Don't attempt to install develop and default packages if Pipfile is missing if not project.pipfile_exists and not packages and dev: click.echo("Could not find Pipfile.", err=True) + sys.exit(1) concurrent = not sequential # Ensure that virtualenv is available. ensure_project( From 167c417183a3a747965ceb9fb0ff2b1da9b3696a Mon Sep 17 00:00:00 2001 From: Jeremy Carroll Date: Sun, 21 Oct 2018 21:56:15 -0700 Subject: [PATCH 026/218] Documenting how I got the tests to run --- docs/dev/contributing.rst | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 4cbb9e3f..ac36a0f7 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -67,7 +67,7 @@ Steps for Submitting Code When contributing code, you'll want to follow this checklist: 1. Fork the repository on GitHub. -2. Run the tests to confirm they all pass on your system. If they don't, you'll +2. `Run the tests`_ to confirm they all pass on your system. If they don't, you'll need to investigate why they fail. If you're unable to diagnose this yourself, raise it as a bug report by following the guidelines in this document: :ref:`bug-reports`. @@ -120,3 +120,38 @@ hasn't been reported before. Duplicate bug reports are a huge drain on the time of other contributors, and should be avoided as much as possible. .. _GitHub issues: https://github.com/pypa/pipenv/issues + +Run the tests +------------- + +Three ways of running the tests are as follows: + +1. ``make test`` (which uses ``docker``) +2. ``./run-tests.sh`` +3. Using pipenv:: + + pipenv install --dev + pipenv run py.test + +For the last two, it is important that your environment is setup correctly, and +this may take some work, for example, on a specific Mac installation, the following +steps may be needed:: + + # Make sure the tests can access github + if [ "$SSH_AGENT_PID" = "" ] + then + eval `ssh-agent` + ssh-add + fi + + # Use unix like utilities, installed with brew, + # e.g. brew install coreutils + for d in /usr/local/opt/*/libexec/gnubin /usr/local/opt/python/libexec/bin + do + [[ ":$PATH:" != *":$d:"* ]] && PATH="$d:${PATH}" + done + + export PATH + + # PIP_FIND_LINKS currently broken + unset PIP_FIND_LINKS From cb5e1bd9d5ed4e95b3d954cb8368e5f0dbbb2f80 Mon Sep 17 00:00:00 2001 From: Jeremy Carroll Date: Sun, 21 Oct 2018 21:57:28 -0700 Subject: [PATCH 027/218] Ignore pycharm project --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 2cee5455..4809d065 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,9 @@ venv.bak/ .spyderproject .spyproject +# PyCharm project settings +.idea + # Rope project settings .ropeproject From dd8c61addbab7c8f3abc7a7e6ced6e3c80c57099 Mon Sep 17 00:00:00 2001 From: Jeremy Carroll Date: Mon, 22 Oct 2018 07:03:47 -0700 Subject: [PATCH 028/218] More specific comment about PIP_FIND_LINKS --- docs/dev/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index ac36a0f7..effbec1f 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -153,5 +153,5 @@ steps may be needed:: export PATH - # PIP_FIND_LINKS currently broken + # PIP_FIND_LINKS currently breaks test_uninstall.py unset PIP_FIND_LINKS From d923a58ca7d9340703f1856f256e5ba7cb6b4ab9 Mon Sep 17 00:00:00 2001 From: Jeremy Carroll Date: Tue, 23 Oct 2018 21:55:11 -0700 Subject: [PATCH 029/218] Correction from PR comment --- docs/dev/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index effbec1f..42df6782 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -131,7 +131,7 @@ Three ways of running the tests are as follows: 3. Using pipenv:: pipenv install --dev - pipenv run py.test + pipenv run pytest For the last two, it is important that your environment is setup correctly, and this may take some work, for example, on a specific Mac installation, the following From 735119e63bf15a114b6f10ae9f77bd7610205b51 Mon Sep 17 00:00:00 2001 From: Jeremy Carroll Date: Wed, 24 Oct 2018 06:55:45 -0700 Subject: [PATCH 030/218] Added run-tests.bat as alternative to run-tests.sh --- docs/dev/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 42df6782..db7f14cc 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -127,7 +127,7 @@ Run the tests Three ways of running the tests are as follows: 1. ``make test`` (which uses ``docker``) -2. ``./run-tests.sh`` +2. ``./run-tests.sh`` or ``run-tests.bat`` 3. Using pipenv:: pipenv install --dev From ceebf21b9cddc899c92121201578ffc4a5e71153 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 30 Oct 2018 01:35:43 -0400 Subject: [PATCH 031/218] Add news entry for #3074 Signed-off-by: Dan Ryan --- news/3074.doc.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/3074.doc.rst diff --git a/news/3074.doc.rst b/news/3074.doc.rst new file mode 100644 index 00000000..85feefd5 --- /dev/null +++ b/news/3074.doc.rst @@ -0,0 +1 @@ +Expanded development and testing documentation for contributors to get started. From 3381f178a977baecefb96f8d46a95879008e4896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dra=C5=BEen=20Lu=C4=8Danin?= Date: Tue, 23 Oct 2018 10:07:17 +0200 Subject: [PATCH 032/218] pipenv open custom command help --- pipenv/cli/command.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 1ce9fee9..af1282b0 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -533,7 +533,13 @@ def graph(bare=False, json=False, json_tree=False, reverse=False): @argument("module", nargs=1) @pass_state def run_open(state, module, *args, **kwargs): - """View a given module in your editor.""" + """View a given module in your editor. + + This uses the EDITOR environment variable. You can temporarily override it, + for example: + + EDITOR=atom pipenv open requests + """ from ..core import which, ensure_project # Ensure that virtualenv is available. From 1ad684a5ab22439997baa63713425820b0df5d56 Mon Sep 17 00:00:00 2001 From: jacrotts Date: Thu, 18 Oct 2018 22:33:24 -0500 Subject: [PATCH 033/218] Implement some behavior for --bare in sync and clean, and add bare as an option to do_clean in cli.py --- pipenv/cli/command.py | 1 + pipenv/core.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index af1282b0..02bd5dc9 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -596,6 +596,7 @@ def sync( @cli.command(short_help="Uninstalls all packages not specified in Pipfile.lock.") +@option("--bare", is_flag=True, default=False, help="Minimal output.") @option("--dry-run", is_flag=True, default=False, help="Just output unneeded packages.") @verbose_option @three_option diff --git a/pipenv/core.py b/pipenv/core.py index b23534d6..6129dc15 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2582,7 +2582,8 @@ def do_sync( deploy=deploy, system=system, ) - click.echo(crayons.green("All dependencies are now up-to-date!")) + if not bare: + click.echo(crayons.green("All dependencies are now up-to-date!")) def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None): @@ -2611,14 +2612,15 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro )] failure = False for apparent_bad_package in installed_package_names: - if dry_run: + if dry_run and not bare: click.echo(apparent_bad_package) else: - click.echo( - crayons.white( - fix_utf8("Uninstalling {0}…".format(repr(apparent_bad_package))), bold=True + if not bare: + click.echo( + crayons.white( + fix_utf8("Uninstalling {0}…".format(repr(apparent_bad_package))), bold=True + ) ) - ) # Uninstall the package. c = delegator.run( "{0} uninstall {1} -y".format(which_pip(), apparent_bad_package) From 64f4b4b04297f18fbd6d1a73864763f584611347 Mon Sep 17 00:00:00 2001 From: jacrotts Date: Thu, 18 Oct 2018 23:24:36 -0500 Subject: [PATCH 034/218] Add news fragment for 3041.feature --- news/3041.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/3041.feature diff --git a/news/3041.feature b/news/3041.feature new file mode 100644 index 00000000..edbf7fb0 --- /dev/null +++ b/news/3041.feature @@ -0,0 +1 @@ +--bare now has effect on sync and clean. From ed6e328da0460a8b521de5f53f39daef060a4eaf Mon Sep 17 00:00:00 2001 From: jacrotts Date: Fri, 19 Oct 2018 00:20:04 -0500 Subject: [PATCH 035/218] Change news fragment to better reflect changes to sync --- news/3041.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/3041.feature b/news/3041.feature index edbf7fb0..4a72e216 100644 --- a/news/3041.feature +++ b/news/3041.feature @@ -1 +1 @@ ---bare now has effect on sync and clean. +--bare now has an effect on clean, and use sync's bare option to reduce output. From e7d6f4b901c902f9f80cddb0f4561ce51e9f8a93 Mon Sep 17 00:00:00 2001 From: jacrotts Date: Fri, 19 Oct 2018 00:36:40 -0500 Subject: [PATCH 036/218] Edit a word in news fragment --- news/3041.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/3041.feature b/news/3041.feature index 4a72e216..79a1d5de 100644 --- a/news/3041.feature +++ b/news/3041.feature @@ -1 +1 @@ ---bare now has an effect on clean, and use sync's bare option to reduce output. +--bare now has an effect on clean, and sync's bare option is now used to reduce output. From 9ff61c777bdcdc87053ae827a6f45e15d1016aaf Mon Sep 17 00:00:00 2001 From: jxltom Date: Mon, 8 Oct 2018 11:29:49 +0800 Subject: [PATCH 037/218] Add tests for locking editable packages with ref --- tests/integration/test_lock.py | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 7743804d..2b520808 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -369,6 +369,42 @@ requests = {git = "https://github.com/requests/requests.git", ref = "master", ed assert c.return_code == 0 +@pytest.mark.lock +@pytest.mark.vcs +@pytest.mark.needs_internet +def test_lock_editable_vcs_with_ref_in_git(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[packages] +requests = {git = "https://github.com/requests/requests.git@883caaf", editable = true} + """.strip()) + c = p.pipenv('lock') + assert c.return_code == 0 + assert p.lockfile['default']['requests']['git'] == 'https://github.com/requests/requests.git' + assert p.lockfile['default']['requests']['ref'] == '883caaf145fbe93bd0d208a6b864de9146087312' + c = p.pipenv('install') + assert c.return_code == 0 + + +@pytest.mark.lock +@pytest.mark.vcs +@pytest.mark.needs_internet +def test_lock_editable_vcs_with_ref(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[packages] +requests = {git = "https://github.com/requests/requests.git", ref = "883caaf", editable = true} + """.strip()) + c = p.pipenv('lock') + assert c.return_code == 0 + assert p.lockfile['default']['requests']['git'] == 'https://github.com/requests/requests.git' + assert p.lockfile['default']['requests']['ref'] == '883caaf145fbe93bd0d208a6b864de9146087312' + c = p.pipenv('install') + assert c.return_code == 0 + + @pytest.mark.extras @pytest.mark.lock @pytest.mark.vcs From 77f97aa058f428611ab11b68b927130855fdb545 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 10 Oct 2018 22:54:02 -0500 Subject: [PATCH 038/218] Set PIPENV_ACTIVE when running pipenv run --- news/2996.trivial | 1 + pipenv/core.py | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 news/2996.trivial diff --git a/news/2996.trivial b/news/2996.trivial new file mode 100644 index 00000000..dbe1006a --- /dev/null +++ b/news/2996.trivial @@ -0,0 +1 @@ +Set `PIPENV_ACTIVE=1` when running `pipenv run`. This is what `pipenv shell` already does. \ No newline at end of file diff --git a/pipenv/core.py b/pipenv/core.py index 6129dc15..1b9c5ade 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2284,6 +2284,8 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): # Ensure that virtualenv is available. ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) + # Set an environment variable, so we know we're in the environment. + os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") load_dot_env() # Activate virtualenv under the current interpreter's environment inline_activate_virtual_environment() From 3140ee6f0b4237b1b5a7d5b4db4e0173d9f10290 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 30 Oct 2018 01:58:15 -0400 Subject: [PATCH 039/218] Exclude setup_path from pipfile Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/__init__.py | 2 +- pipenv/vendor/requirementslib/models/requirements.py | 2 +- pipenv/vendor/vendor.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 8ceccd79..bcbff09e 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.2.2' +__version__ = '1.2.3' import logging diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 08eb5618..e92fe8ab 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -649,7 +649,7 @@ class VCSRequirement(FileRequirement): # Remove potential ref in the end of uri after ref is parsed if "@" in self.link.show_url and "@" in self.uri: uri, ref = self.uri.rsplit("@", 1) - if ref in self.ref: + if self.ref and ref in self.ref: self.uri = uri yield vcsrepo diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 18352c2e..2e00af0c 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -27,7 +27,7 @@ requests==2.20.0 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.2.2 +requirementslib==1.2.3 attrs==18.2.0 distlib==0.2.8 packaging==18.0 From 150ec74f398549d87c8dbfcecac4525b40ea844e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 30 Oct 2018 11:32:05 -0400 Subject: [PATCH 040/218] Fix requirementslib bug Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/requirements.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index e92fe8ab..28d90799 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -649,7 +649,8 @@ class VCSRequirement(FileRequirement): # Remove potential ref in the end of uri after ref is parsed if "@" in self.link.show_url and "@" in self.uri: uri, ref = self.uri.rsplit("@", 1) - if self.ref and ref in self.ref: + checkout = self.req.revision + if checkout and ref in checkout: self.uri = uri yield vcsrepo From 3381790ed13d360edb54c1699a606e67f92dee70 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 30 Oct 2018 11:44:17 -0400 Subject: [PATCH 041/218] Fix unicode encoding to replace non-ascii chars - Drops any unmapped non-ascii characters on non-utf8 systems - Fixes #3131 Signed-off-by: Dan Ryan --- news/3131.bugfix.rst | 1 + pipenv/_compat.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 news/3131.bugfix.rst diff --git a/news/3131.bugfix.rst b/news/3131.bugfix.rst new file mode 100644 index 00000000..6eaa4bdd --- /dev/null +++ b/news/3131.bugfix.rst @@ -0,0 +1 @@ +Added additional logic for ignoring and replacing non-ascii characters when formatting console output on non-UTF-8 systems. diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 6cc4c732..d19bc514 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -379,11 +379,12 @@ def decode_output(output): return output try: output = output.encode(DEFAULT_ENCODING) - except (AttributeError, UnicodeDecodeError): + except (AttributeError, UnicodeDecodeError, UnicodeEncodeError): if six.PY2: output = unicode.translate(vistir.misc.to_text(output), UNICODE_TO_ASCII_TRANSLATION_MAP) else: output = output.translate(UNICODE_TO_ASCII_TRANSLATION_MAP) - output = output.decode(DEFAULT_ENCODING) + output = output.encode(DEFAULT_ENCODING, "replace") + return vistir.misc.to_text(output, encoding=DEFAULT_ENCODING, errors="replace") return output From 2835ff5d46550c35c9500471cbe711785dd1b2e6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 30 Oct 2018 19:16:27 -0400 Subject: [PATCH 042/218] Fix virtualenv path derivations - Fix inadvertent occasional global installation of files - Fix inadvertent occcasional global removal of files - Fix empty output from `pipenv update --outdated` - Fixes #2828 - Fixes #3113 - Fixes #3047 - Fixes #3055 Signed-off-by: Dan Ryan --- news/2828.feature.rst | 1 + news/3047.bugfix.rst | 1 + news/3055.bugfix.rst | 1 + news/3113.bugfix.rst | 2 +- pipenv/__init__.py | 1 + pipenv/core.py | 99 +++++++++++++++++++++++++++++++---------- pipenv/project.py | 100 +++++++++++++++++++++++++++++++++++++----- pipenv/utils.py | 16 ++++++- 8 files changed, 185 insertions(+), 36 deletions(-) create mode 100644 news/2828.feature.rst create mode 100644 news/3047.bugfix.rst create mode 100644 news/3055.bugfix.rst diff --git a/news/2828.feature.rst b/news/2828.feature.rst new file mode 100644 index 00000000..688c47ee --- /dev/null +++ b/news/2828.feature.rst @@ -0,0 +1 @@ +Added additional output to ``pipenv update --outdated`` to indicate that the operation succeded and all packages were already up to date. diff --git a/news/3047.bugfix.rst b/news/3047.bugfix.rst new file mode 100644 index 00000000..6c44bd42 --- /dev/null +++ b/news/3047.bugfix.rst @@ -0,0 +1 @@ +Fixed a virtualenv creation issue which could cause new virtualenvs to inadvertently attempt to read and write to global site packages. diff --git a/news/3055.bugfix.rst b/news/3055.bugfix.rst new file mode 100644 index 00000000..7b2a0fa1 --- /dev/null +++ b/news/3055.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue with virtualenv path derivation which could cause errors, particularly for users on WSL bash. diff --git a/news/3113.bugfix.rst b/news/3113.bugfix.rst index af43b87d..75ee6de4 100644 --- a/news/3113.bugfix.rst +++ b/news/3113.bugfix.rst @@ -1 +1 @@ -Fixed an issue resolving virtualenv paths for users without ``platlib`` values on their systems. +Fixed an issue which caused ``pipenv clean`` to sometimes clean packages from the base ``site-packages`` folder or fail entirely. diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 6b8ddf66..f8a1a8b3 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -22,6 +22,7 @@ from pipenv.vendor.urllib3.exceptions import DependencyWarning from pipenv.vendor.vistir.compat import ResourceWarning, fs_str warnings.filterwarnings("ignore", category=DependencyWarning) warnings.filterwarnings("ignore", category=ResourceWarning) +warnings.filterwarnings("ignore", category=UserWarning) if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): if sys.stdout.isatty() and sys.stderr.isatty(): diff --git a/pipenv/core.py b/pipenv/core.py index 1b9c5ade..8a6d21c2 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -41,7 +41,8 @@ from .utils import ( rmtree, clean_resolved_dep, parse_indexes, - escape_cmd + escape_cmd, + fix_venv_site ) from . import environments, pep508checker, progress from .environments import ( @@ -1705,6 +1706,7 @@ def do_py(system=False): def do_outdated(pypi_mirror=None): + # TODO: Allow --skip-lock here? from .vendor.requirementslib.models.requirements import Requirement packages = {} @@ -1729,6 +1731,9 @@ def do_outdated(pypi_mirror=None): outdated.append( (package, updated_packages[norm_name], packages[package]) ) + if not outdated: + click.echo(crayons.green("All packages are up to date!", bold=True)) + sys.exit(0) for package, new_version, old_version in outdated: click.echo( "Package {0!r} out-of-date: {1!r} installed, {2!r} available.".format( @@ -2062,6 +2067,7 @@ def do_uninstall( ): from .environments import PIPENV_USE_SYSTEM from .vendor.requirementslib.models.requirements import Requirement + from .vendor.packaging.utils import canonicalize_name # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: @@ -2074,6 +2080,24 @@ def do_uninstall( Requirement.from_line("-e {0}".format(p)).name for p in editable_packages if p ] package_names = [p for p in packages if p] + editable_pkgs + installed_package_names = set([ + canonicalize_name(pkg.project_name) for pkg in project.get_installed_packages() + ]) + # Intelligently detect if --dev should be used or not. + if project.lockfile_exists: + develop = set( + [canonicalize_name(k) for k in project.lockfile_content["develop"].keys()] + ) + default = set( + [canonicalize_name(k) for k in project.lockfile_content["default"].keys()] + ) + else: + develop = set( + [canonicalize_name(k) for k in project.dev_packages.keys()] + ) + default = set( + [canonicalize_name(k) for k in project.packages.keys()] + ) pipfile_remove = True # Un-install all dependencies, if --all was provided. if all is True: @@ -2084,7 +2108,7 @@ def do_uninstall( return # Uninstall [dev-packages], if --dev was provided. if all_dev: - if "dev-packages" not in project.parsed_pipfile: + if "dev-packages" not in project.parsed_pipfile and not develop: click.echo( crayons.normal( "No {0} to uninstall.".format(crayons.red("[dev-packages]")), @@ -2097,40 +2121,64 @@ def do_uninstall( fix_utf8("Un-installing {0}…".format(crayons.red("[dev-packages]"))), bold=True ) ) - package_names = project.dev_packages.keys() if packages is False and editable_packages is False and not all_dev: click.echo(crayons.red("No package provided!"), err=True) return 1 - for package_name in package_names: - click.echo(fix_utf8("Un-installing {0}…".format(crayons.green(package_name)))) - cmd = "{0} uninstall {1} -y".format( - escape_grouped_arguments(which_pip(allow_global=system)), package_name + fix_venv_site(project.env_paths["lib"]) + # Remove known "bad packages" from the list. + for bad_package in BAD_PACKAGES: + if canonicalize_name(bad_package) in package_names: + if environments.is_verbose(): + click.echo("Ignoring {0}.".format(repr(bad_package)), err=True) + del package_names[package_names.index( + canonicalize_name(bad_package) + )] + used_packages = (develop | default) & installed_package_names + failure = False + packages_to_remove = set() + if all_dev: + packages_to_remove |= develop & installed_package_names + package_names = set([canonicalize_name(pkg_name) for pkg_name in package_names]) + packages_to_remove = package_names & used_packages + for package_name in packages_to_remove: + click.echo( + crayons.white( + fix_utf8("Uninstalling {0}…".format(repr(package_name))), bold=True + ) ) + # Uninstall the package. + cmd = "{0} uninstall {1} -y".format( + escape_grouped_arguments(which_pip()), package_name + ) if environments.is_verbose(): click.echo("$ {0}".format(cmd)) c = delegator.run(cmd) click.echo(crayons.blue(c.out)) - if pipfile_remove: - in_packages = project.get_package_name_in_pipfile(package_name, dev=False) - in_dev_packages = project.get_package_name_in_pipfile( - package_name, dev=True - ) - if not in_dev_packages and not in_packages: - click.echo( - "No package {0} to remove from Pipfile.".format( - crayons.green(package_name) - ) + if c.return_code != 0: + failure = True + else: + if pipfile_remove: + in_packages = project.get_package_name_in_pipfile(package_name, dev=False) + in_dev_packages = project.get_package_name_in_pipfile( + package_name, dev=True ) - continue + if not in_dev_packages and not in_packages: + click.echo( + "No package {0} to remove from Pipfile.".format( + crayons.green(package_name) + ) + ) + continue - click.echo( - fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name))) - ) - # Remove package from both packages and dev-packages. - project.remove_package_from_pipfile(package_name, dev=True) - project.remove_package_from_pipfile(package_name, dev=False) + click.echo( + fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name))) + ) + # Remove package from both packages and dev-packages. + project.remove_package_from_pipfile(package_name, dev=True) + project.remove_package_from_pipfile(package_name, dev=False) if lock: do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) + sys.exit(int(failure)) def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror=None): @@ -2593,6 +2641,9 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro from packaging.utils import canonicalize_name ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) ensure_lockfile(pypi_mirror=pypi_mirror) + # Make sure that the virtualenv's site packages are configured correctly + # otherwise we may end up removing from the global site packages directory + fix_venv_site(project.env_paths["lib"]) installed_package_names = [ canonicalize_name(pkg.project_name) for pkg in project.get_installed_packages() ] diff --git a/pipenv/project.py b/pipenv/project.py index d25dba9a..eabd7c82 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -5,6 +5,7 @@ import os import re import sys import base64 +import itertools import fnmatch import hashlib import contoml @@ -33,6 +34,7 @@ from .utils import ( get_workon_home, is_virtual_environment, looks_like_dir, + sys_version ) from .environments import ( PIPENV_MAX_DEPTH, @@ -42,6 +44,7 @@ from .environments import ( PIPENV_TEST_INDEX, PIPENV_PYTHON, PIPENV_DEFAULT_PYTHON_VERSION, + PIPENV_CACHE_DIR ) from requirementslib.utils import is_vcs @@ -301,9 +304,9 @@ class Project(object): user_site = site.USER_SITE search_locations = [site_packages, user_site] for site_directory in search_locations: - egg = os.path.join(site_directory, search_filename) - if os.path.isfile(egg): - return egg + egg = os.path.join(site_directory, search_filename) + if os.path.isfile(egg): + return egg def locate_dist(self, dist): location = self.find_egg(dist) @@ -325,6 +328,71 @@ class Project(object): packages = [pkg for pkg in packages] return packages + def get_package_info(self): + from .utils import prepare_pip_source_args + from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder + index_urls = [source.get("url") for source in self.sources] + + class PipCommand(Command): + name = "PipCommand" + + dependency_links = [] + packages = self.get_installed_packages() + # This code is borrowed from pip's current implementation + for dist in packages: + if dist.has_metadata('dependency_links.txt'): + dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) + + pip_command = PipCommand() + index_opts = cmdoptions.make_option_group( + index_group, pip_command.parser + ) + cmd_opts = pip_command.cmd_opts + pip_command.parser.insert_option_group(0, index_opts) + pip_command.parser.insert_option_group(0, cmd_opts) + pip_args = prepare_pip_source_args(self.sources, []) + pip_options, _ = pip_command.parser.parse_args(pip_args) + pip_options.cache_dir = PIPENV_CACHE_DIR + pip_options.pre = self.settings.get("pre", False) + with pip_command._build_session(pip_options) as session: + finder = PackageFinder( + find_links=pip_options.find_links, + index_urls=index_urls, allow_all_prereleases=pip_options.pre, + trusted_hosts=pip_options.trusted_hosts, + process_dependency_links=pip_options.process_dependency_links, + session=session + ) + finder.add_dependency_links(dependency_links) + + for dist in packages: + typ = 'unknown' + all_candidates = finder.find_all_candidates(dist.key) + if not pip_options.pre: + # Remove prereleases + all_candidates = [ + candidate for candidate in all_candidates + if not candidate.version.is_prerelease + ] + + if not all_candidates: + continue + best_candidate = max(all_candidates, key=finder._candidate_sort_key) + remote_version = best_candidate.version + if best_candidate.location.is_wheel: + typ = 'wheel' + else: + typ = 'sdist' + # This is dirty but makes the rest of the code much cleaner + dist.latest_version = remote_version + dist.latest_filetype = typ + yield dist + + def get_outdated_packages(self): + return [ + pkg for pkg in self.get_package_info() + if pkg.latest_version._version > pkg.parsed_version._version + ] + @classmethod def _sanitize(cls, name): # Replace dangerous characters into '_'. The length of the sanitized @@ -975,13 +1043,19 @@ class Project(object): # Return whether or not values have been changed. return changed_values + @property + def py_version(self): + py_path = self.which("python") + version = python_version(py_path) + return version + @property def _pyversion(self): include_dir = vistir.compat.Path(self.virtualenv_location) / "include" python_path = next((x for x in include_dir.iterdir() if x.name.startswith("python")), None) if python_path: - python_version = python_path.name.replace("python", "") - py_version_short, abiflags = python_version[:3], python_version[3:] + py_version = python_path.name.replace("python", "") + py_version_short, abiflags = py_version[:3], py_version[3:] return {"py_version_short": py_version_short, "abiflags": abiflags} return {} @@ -990,8 +1064,10 @@ class Project(object): location = self.virtualenv_location if self.virtualenv_location else sys.prefix prefix = vistir.compat.Path(location) import importlib + py_version = tuple([int(v) for v in self.py_version.split(".")]) try: - _virtualenv = importlib.import_module("virtualenv") + with sys_version(py_version): + _virtualenv = importlib.import_module("virtualenv") except ImportError: with vistir.contextmanagers.temp_path(): from string import Formatter @@ -1015,9 +1091,11 @@ class Project(object): sys.path = [ os.path.join(sysconfig._INSTALL_SCHEMES[scheme][lib_key], "site-packages"), ] + sys.path - six.reload_module(importlib) - _virtualenv = importlib.import_module("virtualenv") - home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix()) + with sys_version(py_version): + six.reload_module(importlib) + _virtualenv = importlib.import_module("virtualenv") + with sys_version(py_version): + home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix()) paths = { "lib": lib, "include": inc, @@ -1031,8 +1109,10 @@ class Project(object): @cached_property def finders(self): from .vendor.pythonfinder import Finder + scripts_dirname = "Scripts" if os.name == "nt" else "bin" + scripts_dir = os.path.join(self.virtualenv_location, scripts_dirname) finders = [ - Finder(path=self.env_paths["scripts"], global_search=gs, system=False) + Finder(path=scripts_dir, global_search=gs, system=False) for gs in (False, True) ] return finders diff --git a/pipenv/utils.py b/pipenv/utils.py index c0979863..33e930d6 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -372,7 +372,7 @@ def venv_resolve_deps( result = None while True: try: - result = c.expect(u"\n", timeout=-1) + result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT) except (EOF, TIMEOUT): pass if result is None: @@ -1341,3 +1341,17 @@ def fix_venv_site(venv_lib_dir): fp.write(site_contents) # Make sure bytecode is up-to-date too. assert compileall.compile_file(str(site_py), quiet=1, force=True) + + +@contextmanager +def sys_version(version_tuple): + """ + Set a temporary sys.version_info tuple + + :param version_tuple: a fake sys.version_info tuple + """ + + old_version = sys.version_info + sys.version_info = version_tuple + yield + sys.version_info = old_version From 084701769768cf46e3408aa1b62c13018c6f6a94 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 31 Oct 2018 09:52:58 +0900 Subject: [PATCH 043/218] Rewrite venv discovery logic --- news/3134.bugfix.rst | 1 + pipenv/project.py | 37 ++++++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 11 deletions(-) create mode 100644 news/3134.bugfix.rst diff --git a/news/3134.bugfix.rst b/news/3134.bugfix.rst new file mode 100644 index 00000000..9193321a --- /dev/null +++ b/news/3134.bugfix.rst @@ -0,0 +1 @@ +Fix virtual environment discovery when `PIPENV_VENV_IN_PROJECT` is set, but the in-project `.venv` is a file. diff --git a/pipenv/project.py b/pipenv/project.py index d25dba9a..29be6d8c 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -269,18 +269,33 @@ class Project(object): return False def get_location_for_virtualenv(self): - if self.is_venv_in_project(): - return os.path.join(self.project_directory, ".venv") + # If there's no project yet, set location based on config. + if not self.project_directory: + if self.is_venv_in_project(): + return os.path.abspath(".venv") + return str(get_workon_home().joinpath(self.virtualenv_name)) - name = self.virtualenv_name - if self.project_directory: - venv_path = os.path.join(self.project_directory, ".venv") - if os.path.exists(venv_path) and not os.path.isdir(".venv"): - with io.open(venv_path, "r") as f: - name = f.read().strip() - # Assume file's contents is a path if it contains slashes. - if looks_like_dir(name): - return vistir.compat.Path(name).absolute().as_posix() + dot_venv = os.path.join(self.project_directory, ".venv") + + # If there's no .venv in project root, set location based on config. + if not os.path.exists(dot_venv): + if self.is_venv_in_project(): + return dot_venv + return str(get_workon_home().joinpath(self.virtualenv_name)) + + # If .venv in project root is a directory, use it. + if os.path.isdir(dot_venv): + return dot_venv + + # Now we assume .venv in project root is a file. Use its content. + with io.open(dot_venv) as f: + name = f.read().strip() + + # If content looks like a path, use it as a relative path. + # Otherwise use directory named after content in WORKON_HOME. + if looks_like_dir(name): + path = vistir.compat.Path(self.project_directory, name) + return path.absolute().as_posix() return str(get_workon_home().joinpath(name)) @property From b57240dd45f69859fba3244dc9f642702a700e21 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 30 Oct 2018 22:30:08 -0400 Subject: [PATCH 044/218] Add messaging for skipped updates - Fix removal of incorrect packages for pipfiles with comments on preceding lines - Improved error handling and added a framework for exception handling - Fixes #2411 - Fixes #3099 - Fixes #2885 - Fixes #1977 Signed-off-by: Dan Ryan --- news/1977.feature.rst | 1 + news/2411.feature.rst | 1 + news/2885.bugfix.rst | 1 + news/3099.bugfix.rst | 1 + pipenv/core.py | 143 ++++++++++++++++++------------------------ pipenv/exceptions.py | 72 +++++++++++++++++++++ pipenv/project.py | 28 ++++++++- 7 files changed, 161 insertions(+), 86 deletions(-) create mode 100644 news/1977.feature.rst create mode 100644 news/2411.feature.rst create mode 100644 news/2885.bugfix.rst create mode 100644 news/3099.bugfix.rst create mode 100644 pipenv/exceptions.py diff --git a/news/1977.feature.rst b/news/1977.feature.rst new file mode 100644 index 00000000..33274c0f --- /dev/null +++ b/news/1977.feature.rst @@ -0,0 +1 @@ +Improved exceptions and error handling on failures. diff --git a/news/2411.feature.rst b/news/2411.feature.rst new file mode 100644 index 00000000..d99bfca8 --- /dev/null +++ b/news/2411.feature.rst @@ -0,0 +1 @@ +Added improved messaging about available but skipped updates due to dependency conflicts when running ``pipenv update --outdated``. diff --git a/news/2885.bugfix.rst b/news/2885.bugfix.rst new file mode 100644 index 00000000..72a7656c --- /dev/null +++ b/news/2885.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which caused uninstallation to sometimes fail to successfullly remove packages from ``Pipfiles`` with comments on preceding or following lines. diff --git a/news/3099.bugfix.rst b/news/3099.bugfix.rst new file mode 100644 index 00000000..72a7656c --- /dev/null +++ b/news/3099.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which caused uninstallation to sometimes fail to successfullly remove packages from ``Pipfiles`` with comments on preceding or following lines. diff --git a/pipenv/core.py b/pipenv/core.py index 8a6d21c2..2d16c559 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -57,6 +57,7 @@ from .environments import ( SESSION_IS_INTERACTIVE, PIPENV_CACHE_DIR, ) +from . import exceptions # Packages that should be ignored later. BAD_PACKAGES = ( @@ -315,14 +316,10 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): if project.pipfile_is_empty: # Show an error message and exit if system is passed and no pipfile exists if system and not PIPENV_VIRTUALENV: - click.echo( - "{0}: --system is intended to be used for pre-existing Pipfile " - "installation, not installation of specific packages. Aborting.".format( - crayons.red("Warning", bold=True) - ), - err=True, + raise exceptions.PipenvOptionsError( + "--system is intended to be used for pre-existing Pipfile " + "installation, not installation of specific packages. Aborting." ) - sys.exit(1) # If there's a requirements file, but no Pipfile… if project.requirements_exists and not skip_requirements: click.echo( @@ -602,6 +599,8 @@ def ensure_project( system = True if not project.pipfile_exists and not deploy: project.touch_pipfile() + else: + raise exceptions.PipfileNotFound # Skip virtualenv creation when --system was used. if not system: ensure_virtualenv( @@ -802,8 +801,7 @@ def do_install_dependencies( if len(indexes) > 1: extra_indexes = indexes[1:] with vistir.contextmanagers.temp_environ(): - if "PIP_USER" in os.environ: - del os.environ["PIP_USER"] + os.environ["PIP_USER"] = vistir.compat.fs_str("0") c = pip_install( dep, ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), @@ -847,8 +845,7 @@ def do_install_dependencies( if len(indexes) > 1: extra_indexes = indexes[1:] with vistir.contextmanagers.temp_environ(): - if "PIP_USER" in os.environ: - del os.environ["PIP_USER"] + os.environ["PIP_USER"] = vistir.compat.fs_str("0") c = pip_install( dep, ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), @@ -946,13 +943,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): click.echo(crayons.blue("{0}".format(c.out)), err=True) if c.returncode != 0: click.echo(crayons.blue("{0}".format(c.err)), err=True) - click.echo( - u"{0}: Failed to create virtual environment.".format( - crayons.red("Warning", bold=True) - ), - err=True, - ) - sys.exit(1) + raise exceptions.VirtualenvCreationException # Associate project directory with the environment. # This mimics Pew's "setproject". @@ -1019,12 +1010,9 @@ def do_lock( pre = project.settings.get("allow_prereleases") if keep_outdated: if not project.lockfile_exists: - click.echo( - "{0}: Pipfile.lock must exist to use --keep-outdated!".format( - crayons.red("Warning", bold=True) - ) + raise exceptions.PipenvOptionsError( + "Pipfile.lock must exist to use --keep-outdated!" ) - sys.exit(1) cached_lockfile = project.lockfile_content # Create the lockfile. lockfile = project._lockfile @@ -1164,7 +1152,6 @@ def do_lock( def do_purge(bare=False, downloads=False, allow_global=False): """Executes the purge functionality.""" - from .vendor.requirementslib.models.requirements import Requirement if downloads: if not bare: @@ -1172,43 +1159,32 @@ def do_purge(bare=False, downloads=False, allow_global=False): shutil.rmtree(project.download_location) return - freeze = delegator.run( - "{0} freeze".format( - escape_grouped_arguments(which_pip(allow_global=allow_global)) - ) - ).out # Remove comments from the output, if any. installed = [ - line for line in freeze.splitlines() if not line.lstrip().startswith("#") + pep423_name(pkg.project_name) for pkg in project.get_installed_packages() ] # Remove setuptools and friends from installed, if present. for package_name in BAD_PACKAGES: for i, package in enumerate(installed): if package.startswith(package_name): del installed[i] - actually_installed = [] - for package in installed: - try: - dep = Requirement.from_line(package) - except AssertionError: - dep = None - if dep and not dep.is_vcs and not dep.editable: - dep = dep.name - actually_installed.append(dep) if not bare: click.echo( - fix_utf8("Found {0} installed package(s), purging…".format(len(actually_installed))) + fix_utf8("Found {0} installed package(s), purging…".format(len(installed))) ) command = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=allow_global)), - " ".join(actually_installed), + " ".join(installed), ) if environments.is_verbose(): click.echo("$ {0}".format(command)) c = delegator.run(command) + if c.return_code != 0: + raise click.exceptions.Exit(c.return_code) if not bare: click.echo(crayons.blue(c.out)) click.echo(crayons.green("Environment now purged and fresh!")) + return installed def do_init( @@ -1255,7 +1231,7 @@ def do_init( ) ) ) - click.echo(crayons.normal("Aborting deploy.", bold=True), err=True) + raise exceptions.DeployException sys.exit(1) elif (system or allow_global) and not (PIPENV_VIRTUALENV): click.echo( @@ -1291,15 +1267,11 @@ def do_init( # Unless we're in a virtualenv not managed by pipenv, abort if we're # using the system's python. if (system or allow_global) and not (PIPENV_VIRTUALENV): - click.echo( - "{0}: --system is intended to be used for Pipfile installation, " - "not installation of specific packages. Aborting.".format( - crayons.red("Warning", bold=True) - ), - err=True, + raise exceptions.PipenvOptionsError( + "--system is intended to be used for Pipfile installation, " + "not installation of specific packages. Aborting.\n" + "See also: --deploy flag." ) - click.echo("See also: --deploy flag.", err=True) - sys.exit(1) else: click.echo( crayons.normal(fix_utf8("Pipfile.lock not found, creating…"), bold=True), @@ -1708,12 +1680,20 @@ def do_py(system=False): def do_outdated(pypi_mirror=None): # TODO: Allow --skip-lock here? from .vendor.requirementslib.models.requirements import Requirement + from .vendor.packaging.utils import canonicalize_name + from collections import namedtuple packages = {} - results = delegator.run("{0} freeze".format(which("pip"))).out.strip().split("\n") - results = filter(bool, results) - for result in results: - dep = Requirement.from_line(result) + package_info = namedtuple("PackageInfo", ["name", "installed", "available"]) + + installed_packages = project.get_installed_packages() + outdated_packages = { + canonicalize_name(pkg.project_name): package_info + (pkg.project_name, pkg.parsed_version, pkg.latest_version) + for pkg in project.get_outdated_packages() + } + for result in installed_packages: + dep = Requirement.from_line(str(result.as_requirement())) packages.update(dep.as_pipfile()) updated_packages = {} lockfile = do_lock(write=False, pypi_mirror=pypi_mirror) @@ -1724,13 +1704,22 @@ def do_outdated(pypi_mirror=None): except KeyError: pass outdated = [] + skipped = [] for package in packages: norm_name = pep423_name(package) if norm_name in updated_packages: if updated_packages[norm_name] != packages[package]: outdated.append( - (package, updated_packages[norm_name], packages[package]) + package_info(package, updated_packages[norm_name], packages[package]) ) + elif canonicalize_name(package) in outdated_packages: + skipped.append(outdated_packages[canonicalize_name(package)]) + for package, old_version, new_version in skipped: + click.echo(crayons.yellow( + "Skipped Update of Package {0!s}: {1!s} installed, {2!s} available.".format( + package, old_version, new_version + )), err=True + ) if not outdated: click.echo(crayons.green("All packages are up to date!", bold=True)) sys.exit(0) @@ -1968,8 +1957,7 @@ def do_install( with vistir.contextmanagers.temp_environ(), spinner(text="Installing...", spinner_name=environments.PIPENV_SPINNER, nospin=environments.PIPENV_NOSPIN) as sp: - if "PIP_USER" in os.environ: - del os.environ["PIP_USER"] + os.environ["PIP_USER"] = vistir.compat.fs_str("0") try: pkg_requirement = Requirement.from_line(pkg_line) except ValueError as e: @@ -2033,9 +2021,7 @@ def do_install( try: project.add_package_to_pipfile(pkg_requirement, dev) except ValueError as e: - click.echo( - "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), e) - ) + raise exceptions.PipfileException(e) # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) @@ -2076,6 +2062,16 @@ def do_uninstall( # TODO: We probably shouldn't ensure a project exists if the outcome will be to just # install things in order to remove them... maybe tell the user to install first? ensure_project(three=three, python=python, pypi_mirror=pypi_mirror) + # Un-install all dependencies, if --all was provided. + if all: + click.echo( + crayons.normal(fix_utf8("Un-installing all packages from virtualenv…"), bold=True) + ) + removed_packages = do_purge(allow_global=system) + project.remove_packages_from_pipfile(removed_packages) + if lock: + do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) + return editable_pkgs = [ Requirement.from_line("-e {0}".format(p)).name for p in editable_packages if p ] @@ -2099,13 +2095,6 @@ def do_uninstall( [canonicalize_name(k) for k in project.packages.keys()] ) pipfile_remove = True - # Un-install all dependencies, if --all was provided. - if all is True: - click.echo( - crayons.normal(fix_utf8("Un-installing all packages from virtualenv…"), bold=True) - ) - do_purge(allow_global=system) - return # Uninstall [dev-packages], if --dev was provided. if all_dev: if "dev-packages" not in project.parsed_pipfile and not develop: @@ -2121,9 +2110,9 @@ def do_uninstall( fix_utf8("Un-installing {0}…".format(crayons.red("[dev-packages]"))), bold=True ) ) + package_names = develop if packages is False and editable_packages is False and not all_dev: - click.echo(crayons.red("No package provided!"), err=True) - return 1 + raise exceptions.MissingParameter(crayons.red("No package provided!")) fix_venv_site(project.env_paths["lib"]) # Remove known "bad packages" from the list. for bad_package in BAD_PACKAGES: @@ -2216,13 +2205,7 @@ def _inline_activate_virtualenv(): try: activate_this = which("activate_this.py") if not activate_this or not os.path.exists(activate_this): - click.echo(fix_utf8( - "{0}: activate_this.py not found. Your environment is most " - "certainly not activated. Continuing anyway…").format( - crayons.red("Warning", bold=True) - ), err=True, - ) - return + raise exceptions.VirtualenvActivationException() with open(activate_this) as f: code = compile(f.read(), activate_this, "exec") exec(code, dict(__file__=activate_this)) @@ -2602,13 +2585,7 @@ def do_sync( ): # The lock file needs to exist because sync won't write to it. if not project.lockfile_exists: - click.echo( - "{0}: Pipfile.lock is missing! You need to run {1} first.".format( - crayons.red("Error", bold=True), crayons.red("$ pipenv lock", bold=True) - ), - err=True, - ) - return 1 + raise exceptions.LockfileNotFound # Ensure that virtualenv is available if not system. ensure_project( diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py new file mode 100644 index 00000000..11f69dc1 --- /dev/null +++ b/pipenv/exceptions.py @@ -0,0 +1,72 @@ +# -*- coding=utf-8 -*- +from .vendor.click.exceptions import ( + ClickException, + Abort, + Exit, + UsageError, + BadParameter, + FileError, + MissingParameter, + BadOptionUsage +) +from .vendor.click import echo as click_echo +from .core import project, fix_utf8 +from .patched import crayons + + +class PipenvException(ClickException): + pass + + +class PipfileNotFound(ClickException): + message = "{0}: Pipfile is missing! Cannot proceed.".format( + crayons.red("Error", bold=True), + ) + + +class LockfileNotFound(ClickException): + message = "{0}: Pipfile.lock is missing! You need to run {1} first.".format( + crayons.red("Error", bold=True), crayons.red("$ pipenv lock", bold=True) + ) + + +class DeployException(ClickException): + message = crayons.normal("Aborting deploy", bold=True) + + +class PipenvOptionsError(BadOptionUsage): + def format_message(self): + return "{0}: {1}".format(crayons.red("Warning", bold=True), self.message) + + +class PipfileException(FileError): + def __init__(self, hint=None): + hint = "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), hint) + filename = project.pipfile_location + super(PipfileException, self).__init__(filename, hint) + + +class SetupException(ClickException): + pass + + +class VirtualenvException(ClickException): + def __init__(self, message=None): + if not message: + message = ( + "There was an unexpected error while activating your virtualenv. " + "Continuing anyway..." + ) + message = fix_utf8("{0}: {1}".format(crayons.red("Warning", bold=True), message)) + super(VirtualenvException, self).__init__(message) + + +class VirtualenvActivationException(VirtualenvException): + message = ( + "activate_this.py not found. Your environment is most certainly " + "not activated. Continuing anyway…" + ) + + +class VirtualenvCreationException(VirtualenvException): + message = "Failed to create virtual environment." diff --git a/pipenv/project.py b/pipenv/project.py index eabd7c82..9f441fb5 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -627,6 +627,12 @@ class Project(object): lockfile[section][norm_key] = lock_section.pop(key) return lockfile + @property + def _pipfile(self): + from .vendor.requirementslib.models.pipfile import Pipfile as ReqLibPipfile + pf = ReqLibPipfile.load(self.pipfile_location) + return pf + @property def lockfile_location(self): return "{0}.lock".format(self.pipfile_location) @@ -910,10 +916,26 @@ class Project(object): # Read and append Pipfile. name = self.get_package_name_in_pipfile(package_name, dev) key = "dev-packages" if dev else "packages" - p = self.parsed_pipfile + p = self._pipfile if name: - del p[key][name] - self.write_toml(p) + del p.pipfile[key][name] + p.write() + + def remove_packages_from_pipfile(self, packages): + p = self._pipfile + packages = [pep423_name(pkg) for pkg in packages] + deleted_pkgs = [] + for section in ("dev-packages", "packages"): + pipfile_section = self.parsed_pipfile.get(section, {}) + pipfile_packages = [ + pkg_name for pkg_name in pipfile_section.keys() + if pep423_name(pkg_name) in packages + ] + for pkg in pipfile_packages: + deleted_pkgs.append(pkg) + del p.pipfile[section][pkg] + if deleted_pkgs: + p.write() def add_package_to_pipfile(self, package, dev=False): from .vendor.requirementslib import Requirement From 7882912c30cfe91f0b9e644825f1cb2c1fe3e678 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 31 Oct 2018 02:00:02 -0400 Subject: [PATCH 045/218] Fix circular import Signed-off-by: Dan Ryan --- pipenv/_compat.py | 12 ++++++++++++ pipenv/core.py | 18 ------------------ pipenv/exceptions.py | 22 +++++++++++++--------- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 6cc4c732..21996c8b 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -387,3 +387,15 @@ def decode_output(output): output = output.translate(UNICODE_TO_ASCII_TRANSLATION_MAP) output = output.decode(DEFAULT_ENCODING) return output + + +def fix_utf8(text): + if not isinstance(text, six.string_types): + return text + from ._compat import decode_output + try: + text = decode_output(text) + except UnicodeDecodeError: + if six.PY2: + text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) + return text diff --git a/pipenv/core.py b/pipenv/core.py index 2d16c559..7a6e7bb9 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -99,24 +99,6 @@ if PIPENV_COLORBLIND: crayons.disable() -UNICODE_TO_ASCII_TRANSLATION_MAP = { - 8230: u"...", - 8211: u"-" -} - - -def fix_utf8(text): - if not isinstance(text, six.string_types): - return text - from ._compat import decode_output - try: - text = decode_output(text) - except UnicodeDecodeError: - if six.PY2: - text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) - return text - - @contextlib.contextmanager def _spinner(text=None, nospin=None, spinner_name=None): if not text: diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 11f69dc1..3d94efb6 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -1,17 +1,17 @@ # -*- coding=utf-8 -*- +from ._compat import fix_utf8 +from .patched import crayons +from .vendor.click import echo as click_echo from .vendor.click.exceptions import ( - ClickException, Abort, - Exit, - UsageError, + BadOptionUsage, BadParameter, + ClickException, + Exit, FileError, MissingParameter, - BadOptionUsage + UsageError, ) -from .vendor.click import echo as click_echo -from .core import project, fix_utf8 -from .patched import crayons class PipenvException(ClickException): @@ -20,7 +20,7 @@ class PipenvException(ClickException): class PipfileNotFound(ClickException): message = "{0}: Pipfile is missing! Cannot proceed.".format( - crayons.red("Error", bold=True), + crayons.red("Error", bold=True) ) @@ -41,6 +41,8 @@ class PipenvOptionsError(BadOptionUsage): class PipfileException(FileError): def __init__(self, hint=None): + from .core import project + hint = "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), hint) filename = project.pipfile_location super(PipfileException, self).__init__(filename, hint) @@ -57,7 +59,9 @@ class VirtualenvException(ClickException): "There was an unexpected error while activating your virtualenv. " "Continuing anyway..." ) - message = fix_utf8("{0}: {1}".format(crayons.red("Warning", bold=True), message)) + message = fix_utf8( + "{0}: {1}".format(crayons.red("Warning", bold=True), message) + ) super(VirtualenvException, self).__init__(message) From 55cd7bfb467565c7807ba1742490e3f00e2a3618 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 31 Oct 2018 02:05:16 -0400 Subject: [PATCH 046/218] Fix exception Signed-off-by: Dan Ryan --- pipenv/exceptions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 3d94efb6..feb15df3 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -15,7 +15,8 @@ from .vendor.click.exceptions import ( class PipenvException(ClickException): - pass + def __init__(self, message, *args, **kwargs): + super(PipenvException, self).__init__(message, *args, **kwargs) class PipfileNotFound(ClickException): From 2625e8d2f5ffc39b2b139fa81052a3bf0f35e3d7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 31 Oct 2018 18:27:25 -0400 Subject: [PATCH 047/218] More exception cleanup Signed-off-by: Dan Ryan --- pipenv/core.py | 5 +- pipenv/exceptions.py | 163 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 136 insertions(+), 32 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 7a6e7bb9..fcf846c0 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -57,6 +57,7 @@ from .environments import ( SESSION_IS_INTERACTIVE, PIPENV_CACHE_DIR, ) +from ._compat import fix_utf8 from . import exceptions # Packages that should be ignored later. @@ -621,7 +622,7 @@ def ensure_project( err=True, ) else: - click.echo(crayons.red("Deploy aborted."), err=True) + raise exceptions.DeployException sys.exit(1) # Ensure the Pipfile exists. ensure_pipfile( @@ -1499,7 +1500,7 @@ def system_which(command, mult=False): ) assert c.return_code == 0 except AssertionError: - return None if not mult else[] + return None if not mult else [] except TypeError: from .vendor.pythonfinder import Finder finder = Finder() diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index feb15df3..3f8aafdc 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -1,7 +1,11 @@ # -*- coding=utf-8 -*- + +import six + from ._compat import fix_utf8 from .patched import crayons from .vendor.click import echo as click_echo +from .vendor.click._compat import get_text_stderr from .vendor.click.exceptions import ( Abort, BadOptionUsage, @@ -15,63 +19,162 @@ from .vendor.click.exceptions import ( class PipenvException(ClickException): - def __init__(self, message, *args, **kwargs): - super(PipenvException, self).__init__(message, *args, **kwargs) + message = "{0}: {{1}}".format(crayons.red("Error", bold=True)) + + def __init__(self, *args, message=None, **kwargs): + if not message: + message = "Pipenv encountered a problem and had to exit." + extra = kwargs.pop("extra", []) + message = self.message.format(message) + ClickException.__init__(self, message) + self.extra = extra + + def show(self, file=None): + if file is None: + file = get_text_stderr() + if self.extra: + if isinstance(self.extra, six.string_types): + self.extra = [self.extra,] + for extra in self.extra: + click_echo(extra, file=file) + super(PipenvException, self).show(file=file) -class PipfileNotFound(ClickException): - message = "{0}: Pipfile is missing! Cannot proceed.".format( - crayons.red("Error", bold=True) - ) +class PipenvUsageError(UsageError): + message = "{0}: {{1}}".format(crayons.red("Error", bold=True)) + + def __init__(self, *args, message=None, ctx=None, **kwargs): + if not message: + message = "Pipenv encountered a problem and had to exit." + self.message = self.message.format(message) + extra = kwargs.pop("extra", []) + UsageError.__init__(self, message, ctx) + self.extra = extra + + def show(self, file=None): + if file is None: + file = get_text_stderr() + if self.extra: + if isinstance(self.extra, six.string_types): + self.extra = [self.extra,] + for extra in self.extra: + click_echo(extra, file=file) + super(PipenvUsageError, self).show(file=file) -class LockfileNotFound(ClickException): - message = "{0}: Pipfile.lock is missing! You need to run {1} first.".format( - crayons.red("Error", bold=True), crayons.red("$ pipenv lock", bold=True) - ) +class PipenvFileError(FileError): + def __init__(self, filename, message=None, **kwargs): + extra = kwargs.pop("extra", []) + FileError.__init__(self, filename, hint=message, **kwargs) + self.extra = extra + + def show(self, file=None): + if file is None: + file = get_text_stderr() + if self.extra: + if isinstance(self.extra, six.string_types): + self.extra = [self.extra,] + for extra in self.extra: + click_echo(extra, file=file) + super(PipenvException, self).show(file=file) -class DeployException(ClickException): - message = crayons.normal("Aborting deploy", bold=True) +class PipfileNotFound(PipenvFileError): + def __init__(self, extra=None, **kwargs): + extra = kwargs.pop("extra", []) + message = ("Cannot proceed. Please ensure that a Pipfile exists and is located " + "in your project root directory.") + filename = "Pipfile" + PipenvFileError.__init__(filename, message=message, extra=extra, **kwargs) + + +class LockfileNotFound(PipenvFileError): + def __init__(self, extra=None, **kwargs): + extra = kwargs.pop("extra", []) + message = "You need to run {0} before you can continue.".format( + crayons.red("$ pipenv lock", bold=True) + ) + filename = "Pipfile.lock" + PipenvFileError.__init__(self, filename, message=message, extra=extra, **kwargs) + + +class DeployException(PipenvException): + def __init__(self, message=None, **kwargs): + if not message: + message = crayons.normal("Aborting deploy", bold=True) + extra = kwargs.pop("extra", None) + PipenvException.__init__(self, message=message, extra=extra, **kwargs) class PipenvOptionsError(BadOptionUsage): - def format_message(self): - return "{0}: {1}".format(crayons.red("Warning", bold=True), self.message) + def __init__(self, message=None, *args, **kwargs): + extra = kwargs.pop("extra", []) + BadOptionUsage.__init__(self, message, *args, **kwargs) + self.extra = extra + + def show(self, file=None): + if file is None: + file = get_text_stderr() + if self.extra: + if isinstance(self.extra, six.string_types): + self.extra = [self.extra,] + for extra in self.extra: + click_echo(extra, file=file) + click_echo("{0}: {1}".format(crayons.red("Warning", bold=True), self.message)) -class PipfileException(FileError): - def __init__(self, hint=None): +class PipfileException(PipenvFileError): + def __init__(self, hint=None, **kwargs): from .core import project hint = "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), hint) filename = project.pipfile_location - super(PipfileException, self).__init__(filename, hint) + extra = kwargs.pop("extra", []) + PipfileException.__init__(self, filename, hint, extra=extra, **kwargs) -class SetupException(ClickException): - pass +class SetupException(PipenvException): + def __init__(self, message=None, *args, **kwargs): + PipenvException.__init__(message, *args, **kwargs) -class VirtualenvException(ClickException): - def __init__(self, message=None): +class VirtualenvException(PipenvException): + + def __init__(self, message=None, **kwargs): if not message: message = ( "There was an unexpected error while activating your virtualenv. " "Continuing anyway..." ) - message = fix_utf8( - "{0}: {1}".format(crayons.red("Warning", bold=True), message) - ) - super(VirtualenvException, self).__init__(message) + PipenvException.__init__(self, message, **kwargs) + + def show(self, file=None): + if file is None: + file = get_text_stderr() + if self.extra: + if isinstance(self.extra, six.string_types): + self.extra = [self.extra,] + for extra in self.extra: + click_echo(extra, file=file) + click_echo(fix_utf8( + "{0}: {1}".format(crayons.red("Warning", bold=True), self.message) + )) class VirtualenvActivationException(VirtualenvException): - message = ( - "activate_this.py not found. Your environment is most certainly " - "not activated. Continuing anyway…" - ) + def __init__(self, message=None, **kwargs): + if not message: + message = ( + "activate_this.py not found. Your environment is most certainly " + "not activated. Continuing anyway…" + ) + self.message = message + VirtualenvException.__init__(self, message, **kwargs) class VirtualenvCreationException(VirtualenvException): - message = "Failed to create virtual environment." + def __init__(self, message=None, **kwargs): + if not message: + message = "Failed to create virtual environment." + self.message = message + VirtualenvException.__init__(self, message, **kwargs) From 1034022e96877ab07a08c7a2a3a7c4fb1c9f8b8e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 1 Nov 2018 00:51:34 -0400 Subject: [PATCH 048/218] Add improved exception handling and formatter Signed-off-by: Dan Ryan --- pipenv/cli/command.py | 6 ++ pipenv/core.py | 35 +++++---- pipenv/exceptions.py | 167 ++++++++++++++++++++++++++++-------------- 3 files changed, 139 insertions(+), 69 deletions(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 02bd5dc9..40d3f0fe 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -268,7 +268,9 @@ def install( ) @uninstall_options @pass_state +@pass_context def uninstall( + ctx, state, lock=False, all_dev=False, @@ -289,6 +291,7 @@ def uninstall( all=all, keep_outdated=state.installstate.keep_outdated, pypi_mirror=state.pypi_mirror, + ctx=ctx ) if retcode: sys.exit(retcode) @@ -297,7 +300,9 @@ def uninstall( @cli.command(short_help="Generates Pipfile.lock.") @lock_options @pass_state +@pass_context def lock( + ctx, state, **kwargs ): @@ -310,6 +315,7 @@ def lock( do_init(dev=state.installstate.dev, requirements=state.installstate.requirementstxt, pypi_mirror=state.pypi_mirror, pre=state.installstate.pre) do_lock( + ctx=ctx, clear=state.clear, pre=state.installstate.pre, keep_outdated=state.installstate.keep_outdated, diff --git a/pipenv/core.py b/pipenv/core.py index fcf846c0..727de0e8 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -299,7 +299,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): if project.pipfile_is_empty: # Show an error message and exit if system is passed and no pipfile exists if system and not PIPENV_VIRTUALENV: - raise exceptions.PipenvOptionsError( + raise exceptions.PipenvOptionsError("--system", "--system is intended to be used for pre-existing Pipfile " "installation, not installation of specific packages. Aborting." ) @@ -580,10 +580,11 @@ def ensure_project( # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True - if not project.pipfile_exists and not deploy: - project.touch_pipfile() - else: - raise exceptions.PipfileNotFound + if not project.pipfile_exists: + if deploy is True: + raise exceptions.PipfileNotFound + else: + project.touch_pipfile() # Skip virtualenv creation when --system was used. if not system: ensure_virtualenv( @@ -623,7 +624,6 @@ def ensure_project( ) else: raise exceptions.DeployException - sys.exit(1) # Ensure the Pipfile exists. ensure_pipfile( validate=validate, skip_requirements=skip_requirements, system=system @@ -925,8 +925,9 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): block=False, nospin=nospin, env=pip_config) click.echo(crayons.blue("{0}".format(c.out)), err=True) if c.returncode != 0: - click.echo(crayons.blue("{0}".format(c.err)), err=True) - raise exceptions.VirtualenvCreationException + raise exceptions.VirtualenvCreationException( + extra=[crayons.blue("{0}".format(c.err)),] + ) # Associate project directory with the environment. # This mimics Pew's "setproject". @@ -978,6 +979,7 @@ def get_downloads_info(names_map, section): def do_lock( + ctx=None, system=False, clear=False, pre=False, @@ -994,7 +996,8 @@ def do_lock( if keep_outdated: if not project.lockfile_exists: raise exceptions.PipenvOptionsError( - "Pipfile.lock must exist to use --keep-outdated!" + "--keep-outdated", ctx=ctx, + message="Pipfile.lock must exist to use --keep-outdated!" ) cached_lockfile = project.lockfile_content # Create the lockfile. @@ -1162,8 +1165,8 @@ def do_purge(bare=False, downloads=False, allow_global=False): if environments.is_verbose(): click.echo("$ {0}".format(command)) c = delegator.run(command) - if c.return_code != 0: - raise click.exceptions.Exit(c.return_code) + if c.return_code != 0 or c.return_code == 0: + raise exceptions.UninstallError(installed, command, c.out + c.err, 1) if not bare: click.echo(crayons.blue(c.out)) click.echo(crayons.green("Environment now purged and fresh!")) @@ -1250,7 +1253,7 @@ def do_init( # Unless we're in a virtualenv not managed by pipenv, abort if we're # using the system's python. if (system or allow_global) and not (PIPENV_VIRTUALENV): - raise exceptions.PipenvOptionsError( + raise exceptions.PipenvOptionsError("--system", "--system is intended to be used for Pipfile installation, " "not installation of specific packages. Aborting.\n" "See also: --deploy flag." @@ -2033,6 +2036,7 @@ def do_uninstall( all=False, keep_outdated=False, pypi_mirror=None, + ctx=None ): from .environments import PIPENV_USE_SYSTEM from .vendor.requirementslib.models.requirements import Requirement @@ -2046,6 +2050,9 @@ def do_uninstall( # install things in order to remove them... maybe tell the user to install first? ensure_project(three=three, python=python, pypi_mirror=pypi_mirror) # Un-install all dependencies, if --all was provided. + if not any([packages, editable_packages, all_dev, all]): + raise exceptions.MissingParameter(crayons.red("No package provided!"), ctx=ctx, + param_type="parameter") if all: click.echo( crayons.normal(fix_utf8("Un-installing all packages from virtualenv…"), bold=True) @@ -2094,8 +2101,6 @@ def do_uninstall( ) ) package_names = develop - if packages is False and editable_packages is False and not all_dev: - raise exceptions.MissingParameter(crayons.red("No package provided!")) fix_venv_site(project.env_paths["lib"]) # Remove known "bad packages" from the list. for bad_package in BAD_PACKAGES: @@ -2568,7 +2573,7 @@ def do_sync( ): # The lock file needs to exist because sync won't write to it. if not project.lockfile_exists: - raise exceptions.LockfileNotFound + raise exceptions.LockfileNotFound(project.lockfile_location) # Ensure that virtualenv is available if not system. ensure_project( diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 3f8aafdc..86643c8a 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -1,10 +1,17 @@ # -*- coding=utf-8 -*- +import itertools +import sys + +from traceback import format_exception +from pprint import pformat + import six from ._compat import fix_utf8 from .patched import crayons -from .vendor.click import echo as click_echo +from . import environments +from .vendor.click.utils import echo as click_echo from .vendor.click._compat import get_text_stderr from .vendor.click.exceptions import ( Abort, @@ -16,12 +23,36 @@ from .vendor.click.exceptions import ( MissingParameter, UsageError, ) +from .vendor.click.types import Path + + +def handle_exception(exc_type, exception, traceback, hook=sys.excepthook): + if environments.is_verbose() or not issubclass(exc_type, ClickException): + hook(exc_type, exception, traceback) + else: + exc = format_exception(exc_type, exception, traceback) + lines = itertools.chain.from_iterable([l.splitlines() for l in exc]) + lines = list(lines)[-11:-1] + for line in lines: + line = line.strip("'").strip('"').strip("\n").strip() + if not line.startswith("File"): + line = " {0}".format(line) + else: + line = " {0}".format(line) + line = "[pipenv.exceptions.{0!s}]: {1}".format( + exception.__class__.__name__, line + ) + click_echo(fix_utf8(line), err=True) + exception.show() + + +sys.excepthook = handle_exception class PipenvException(ClickException): - message = "{0}: {{1}}".format(crayons.red("Error", bold=True)) + message = "{0}: {{0}}".format(crayons.red("ERROR", bold=True)) - def __init__(self, *args, message=None, **kwargs): + def __init__(self, message=None, **kwargs): if not message: message = "Pipenv encountered a problem and had to exit." extra = kwargs.pop("extra", []) @@ -36,17 +67,22 @@ class PipenvException(ClickException): if isinstance(self.extra, six.string_types): self.extra = [self.extra,] for extra in self.extra: + extra = "[pipenv.exceptions.{0!s}]: {1}".format( + self.__class__.__name__, extra + ) click_echo(extra, file=file) - super(PipenvException, self).show(file=file) + click_echo(fix_utf8("{0}".format(self.message)), file=file) class PipenvUsageError(UsageError): - message = "{0}: {{1}}".format(crayons.red("Error", bold=True)) - def __init__(self, *args, message=None, ctx=None, **kwargs): + def __init__(self, message=None, ctx=None, **kwargs): + formatted_message = "{0}: {1}" + msg_prefix = crayons.red("ERROR:", bold=True) if not message: message = "Pipenv encountered a problem and had to exit." - self.message = self.message.format(message) + message = formatted_message.format(msg_prefix, crayons.white(message, bold=True)) + self.message = message extra = kwargs.pop("extra", []) UsageError.__init__(self, message, ctx) self.extra = extra @@ -54,18 +90,40 @@ class PipenvUsageError(UsageError): def show(self, file=None): if file is None: file = get_text_stderr() + color = None + if self.ctx is not None: + color = self.ctx.color if self.extra: if isinstance(self.extra, six.string_types): self.extra = [self.extra,] for extra in self.extra: + if color: + extra = getattr(crayons, color, "normal") click_echo(extra, file=file) - super(PipenvUsageError, self).show(file=file) + hint = '' + if (self.cmd is not None and + self.cmd.get_help_option(self.ctx) is not None): + hint = ('Try "%s %s" for help.\n' + % (self.ctx.command_path, self.ctx.help_option_names[0])) + if self.ctx is not None: + click_echo(self.ctx.get_usage() + '\n%s' % hint, file=file, color=color) + click_echo(self.message, file=file) class PipenvFileError(FileError): + formatted_message = "{0} {{0}} {{1}}".format( + crayons.red("ERROR:", bold=True) + ) + def __init__(self, filename, message=None, **kwargs): extra = kwargs.pop("extra", []) - FileError.__init__(self, filename, hint=message, **kwargs) + if not message: + message = crayons.white("Please ensure that the file exists!", bold=True) + message = self.formatted_message.format( + crayons.white("{0} not found!".format(filename), bold=True), + message + ) + FileError.__init__(self, filename=filename, hint=message, **kwargs) self.extra = extra def show(self, file=None): @@ -76,66 +134,62 @@ class PipenvFileError(FileError): self.extra = [self.extra,] for extra in self.extra: click_echo(extra, file=file) - super(PipenvException, self).show(file=file) + click_echo(self.message, file=file) class PipfileNotFound(PipenvFileError): - def __init__(self, extra=None, **kwargs): + def __init__(self, filename="Pipfile", extra=None, **kwargs): extra = kwargs.pop("extra", []) - message = ("Cannot proceed. Please ensure that a Pipfile exists and is located " - "in your project root directory.") - filename = "Pipfile" - PipenvFileError.__init__(filename, message=message, extra=extra, **kwargs) + message = ("{0}. {1}".format( + crayons.red("Aborting!", bold=True), + crayons.white("Please ensure that the file exists and is located in your" + " project root directory.", bold=True) + ) + ) + super(PipfileNotFound, self).__init__(filename, message=message, extra=extra, **kwargs) class LockfileNotFound(PipenvFileError): - def __init__(self, extra=None, **kwargs): + def __init__(self, filename="Pipfile.lock", extra=None, **kwargs): extra = kwargs.pop("extra", []) - message = "You need to run {0} before you can continue.".format( - crayons.red("$ pipenv lock", bold=True) + message = "{0} {1} {2}".format( + crayons.white("You need to run", bold=True), + crayons.red("$ pipenv lock", bold=True), + crayons.white("before you can continue.", bold=True) ) - filename = "Pipfile.lock" - PipenvFileError.__init__(self, filename, message=message, extra=extra, **kwargs) + super(LockfileNotFound, self).__init__(filename, message=message, extra=extra, **kwargs) -class DeployException(PipenvException): +class DeployException(PipenvUsageError): def __init__(self, message=None, **kwargs): if not message: message = crayons.normal("Aborting deploy", bold=True) - extra = kwargs.pop("extra", None) - PipenvException.__init__(self, message=message, extra=extra, **kwargs) - - -class PipenvOptionsError(BadOptionUsage): - def __init__(self, message=None, *args, **kwargs): extra = kwargs.pop("extra", []) - BadOptionUsage.__init__(self, message, *args, **kwargs) - self.extra = extra + PipenvUsageError.__init__(message=message, extra=extra, **kwargs) - def show(self, file=None): - if file is None: - file = get_text_stderr() - if self.extra: - if isinstance(self.extra, six.string_types): - self.extra = [self.extra,] - for extra in self.extra: - click_echo(extra, file=file) - click_echo("{0}: {1}".format(crayons.red("Warning", bold=True), self.message)) + +class PipenvOptionsError(PipenvUsageError): + def __init__(self, option_name, message=None, ctx=None, **kwargs): + extra = kwargs.pop("extra", []) + PipenvUsageError.__init__(self, message=message, ctx=ctx, **kwargs) + self.extra = extra + self.option_name = option_name class PipfileException(PipenvFileError): def __init__(self, hint=None, **kwargs): from .core import project - hint = "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), hint) + if not hint: + hint = "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), hint) filename = project.pipfile_location extra = kwargs.pop("extra", []) - PipfileException.__init__(self, filename, hint, extra=extra, **kwargs) + PipenvFileError.__init__(self, filename, hint, extra=extra, **kwargs) class SetupException(PipenvException): - def __init__(self, message=None, *args, **kwargs): - PipenvException.__init__(message, *args, **kwargs) + def __init__(self, message=None, **kwargs): + PipenvException.__init__(message, **kwargs) class VirtualenvException(PipenvException): @@ -148,18 +202,6 @@ class VirtualenvException(PipenvException): ) PipenvException.__init__(self, message, **kwargs) - def show(self, file=None): - if file is None: - file = get_text_stderr() - if self.extra: - if isinstance(self.extra, six.string_types): - self.extra = [self.extra,] - for extra in self.extra: - click_echo(extra, file=file) - click_echo(fix_utf8( - "{0}: {1}".format(crayons.red("Warning", bold=True), self.message) - )) - class VirtualenvActivationException(VirtualenvException): def __init__(self, message=None, **kwargs): @@ -178,3 +220,20 @@ class VirtualenvCreationException(VirtualenvException): message = "Failed to create virtual environment." self.message = message VirtualenvException.__init__(self, message, **kwargs) + + +class UninstallError(PipenvException): + def __init__(self, package, command, return_values, return_code, **kwargs): + extra = [crayons.blue("Attempted to run command: {0}".format( + crayons.yellow("$ {0}".format(command), bold=True) + )),] + extra.extend([crayons.blue(line.strip()) for line in return_values.splitlines()]) + if isinstance(package, (tuple, list)): + package = " ".join(package) + message = "{0} {1}...".format( + crayons.normal("Failed to uninstall package(s)"), + crayons.yellow(package, bold=True) + ) + self.exit_code = return_code + PipenvException.__init__(self, message=message, extra=extra) + self.extra = extra From 83af37a2d153d41a71cc463742d4e0f9e558cc51 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 1 Nov 2018 16:20:45 -0400 Subject: [PATCH 049/218] Fix uninstallation if comments are present Signed-off-by: Dan Ryan --- pipenv/core.py | 65 ++++++++++++++++++---------------- pipenv/project.py | 36 +++++++++++++++---- tests/integration/test_sync.py | 2 +- 3 files changed, 65 insertions(+), 38 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 727de0e8..8c8d7e39 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1165,8 +1165,8 @@ def do_purge(bare=False, downloads=False, allow_global=False): if environments.is_verbose(): click.echo("$ {0}".format(command)) c = delegator.run(command) - if c.return_code != 0 or c.return_code == 0: - raise exceptions.UninstallError(installed, command, c.out + c.err, 1) + if c.return_code != 0: + raise exceptions.UninstallError(installed, command, c.out + c.err, c.return_code) if not bare: click.echo(crayons.blue(c.out)) click.echo(crayons.green("Environment now purged and fresh!")) @@ -2110,48 +2110,53 @@ def do_uninstall( del package_names[package_names.index( canonicalize_name(bad_package) )] - used_packages = (develop | default) & installed_package_names + used_packages = develop | default & installed_package_names failure = False packages_to_remove = set() if all_dev: packages_to_remove |= develop & installed_package_names - package_names = set([canonicalize_name(pkg_name) for pkg_name in package_names]) - packages_to_remove = package_names & used_packages - for package_name in packages_to_remove: + package_names = set([pkg_name for pkg_name in package_names]) + packages_to_remove = [ + pkg_name for pkg_name in packages + if canonicalize_name(pkg_name) in used_packages + ] + for package_name in package_names: click.echo( crayons.white( fix_utf8("Uninstalling {0}…".format(repr(package_name))), bold=True ) ) # Uninstall the package. - cmd = "{0} uninstall {1} -y".format( - escape_grouped_arguments(which_pip()), package_name - ) - if environments.is_verbose(): - click.echo("$ {0}".format(cmd)) - c = delegator.run(cmd) - click.echo(crayons.blue(c.out)) - if c.return_code != 0: - failure = True - else: - if pipfile_remove: - in_packages = project.get_package_name_in_pipfile(package_name, dev=False) - in_dev_packages = project.get_package_name_in_pipfile( - package_name, dev=True - ) - if not in_dev_packages and not in_packages: - click.echo( - "No package {0} to remove from Pipfile.".format( - crayons.green(package_name) - ) + if package_name in packages_to_remove: + cmd = "{0} uninstall {1} -y".format( + escape_grouped_arguments(which_pip()), package_name ) - continue - + if environments.is_verbose(): + click.echo("$ {0}".format(cmd)) + c = delegator.run(cmd) + click.echo(crayons.blue(c.out)) + if c.return_code != 0: + failure = True + if not failure and pipfile_remove: + in_packages = project.get_package_name_in_pipfile(package_name, dev=False) + in_dev_packages = project.get_package_name_in_pipfile( + package_name, dev=True + ) + if not in_dev_packages and not in_packages: click.echo( - fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name))) + "No package {0} to remove from Pipfile.".format( + crayons.green(package_name) + ) ) - # Remove package from both packages and dev-packages. + continue + + click.echo( + fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name))) + ) + # Remove package from both packages and dev-packages. + if in_dev_packages: project.remove_package_from_pipfile(package_name, dev=True) + if in_packages: project.remove_package_from_pipfile(package_name, dev=False) if lock: do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) diff --git a/pipenv/project.py b/pipenv/project.py index 9f441fb5..74debc24 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -916,26 +916,48 @@ class Project(object): # Read and append Pipfile. name = self.get_package_name_in_pipfile(package_name, dev) key = "dev-packages" if dev else "packages" - p = self._pipfile - if name: - del p.pipfile[key][name] + p = self.parsed_pipfile + lines = [l for l in p[key].serialized().splitlines()] + if not any(line.startswith("#") for line in lines) and name: + del p[key][name] + self.write_toml(p) + else: + p = self._pipfile + del p[key][name] p.write() def remove_packages_from_pipfile(self, packages): p = self._pipfile + parsed = self.parsed_pipfile packages = [pep423_name(pkg) for pkg in packages] deleted_pkgs = [] + has_comments_as_lines = False for section in ("dev-packages", "packages"): pipfile_section = self.parsed_pipfile.get(section, {}) + lines = [l for l in p[section].serialized().splitlines()] pipfile_packages = [ pkg_name for pkg_name in pipfile_section.keys() if pep423_name(pkg_name) in packages ] - for pkg in pipfile_packages: - deleted_pkgs.append(pkg) - del p.pipfile[section][pkg] + # The normal toml parser can't handle deleting packages with preceding newlines + is_dev = section == "dev-packages" + if any(line.startswith("#") for line in lines): + has_comments_as_lines = True + for pkg in pipfile_packages: + pkg_name = self.get_package_name_in_pipfile(pkg, dev=is_dev) + deleted_pkgs.append(pkg) + del p.pipfile[section][pkg_name] + # However the alternative parser can't handle inline comment preservation + else: + for pkg in pipfile_packages: + pkg_name = self.get_package_name_in_pipfile(pkg, dev=is_dev) + deleted_pkgs.append(pkg) + del parsed[section][pkg_name] if deleted_pkgs: - p.write() + if has_comments_as_lines: + p.write() + else: + self.write_toml(parsed) def add_package_to_pipfile(self, package, dev=False): from .vendor.requirementslib import Requirement diff --git a/tests/integration/test_sync.py b/tests/integration/test_sync.py index c50e259b..2ef06ddc 100644 --- a/tests/integration/test_sync.py +++ b/tests/integration/test_sync.py @@ -15,7 +15,7 @@ def test_sync_error_without_lockfile(PipenvInstance, pypi): c = p.pipenv('sync') assert c.return_code != 0 - assert 'Pipfile.lock is missing!' in c.err + assert 'Pipfile.lock not found!' in c.err @pytest.mark.sync From 41e637e8108808ed2d92513939178b33d3248cf7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 1 Nov 2018 18:54:14 -0400 Subject: [PATCH 050/218] Fix uninstallation passthrough Signed-off-by: Dan Ryan --- pipenv/core.py | 72 +++++++++++-------- pipenv/project.py | 5 +- .../requirementslib/models/requirements.py | 8 +-- 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 8c8d7e39..85f9b2e9 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1146,21 +1146,20 @@ def do_purge(bare=False, downloads=False, allow_global=False): return # Remove comments from the output, if any. - installed = [ + installed = set([ pep423_name(pkg.project_name) for pkg in project.get_installed_packages() - ] - # Remove setuptools and friends from installed, if present. - for package_name in BAD_PACKAGES: - for i, package in enumerate(installed): - if package.startswith(package_name): - del installed[i] + ]) + bad_pkgs = set([pep423_name(pkg) for pkg in BAD_PACKAGES]) + # Remove setuptools, pip, etc from targets for removal + to_remove = installed - bad_pkgs + if not bare: click.echo( - fix_utf8("Found {0} installed package(s), purging…".format(len(installed))) + fix_utf8("Found {0} installed package(s), purging…".format(len(to_remove))) ) command = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=allow_global)), - " ".join(installed), + " ".join(to_remove), ) if environments.is_verbose(): click.echo("$ {0}".format(command)) @@ -2053,23 +2052,19 @@ def do_uninstall( if not any([packages, editable_packages, all_dev, all]): raise exceptions.MissingParameter(crayons.red("No package provided!"), ctx=ctx, param_type="parameter") - if all: - click.echo( - crayons.normal(fix_utf8("Un-installing all packages from virtualenv…"), bold=True) - ) - removed_packages = do_purge(allow_global=system) - project.remove_packages_from_pipfile(removed_packages) - if lock: - do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) - return editable_pkgs = [ Requirement.from_line("-e {0}".format(p)).name for p in editable_packages if p ] - package_names = [p for p in packages if p] + editable_pkgs + packages = packages + editable_pkgs + package_names = [p for p in packages if p] + package_map = { + canonicalize_name(p): p for p in packages if p + } installed_package_names = set([ canonicalize_name(pkg.project_name) for pkg in project.get_installed_packages() ]) # Intelligently detect if --dev should be used or not. + lockfile_packages = set() if project.lockfile_exists: develop = set( [canonicalize_name(k) for k in project.lockfile_content["develop"].keys()] @@ -2077,6 +2072,7 @@ def do_uninstall( default = set( [canonicalize_name(k) for k in project.lockfile_content["default"].keys()] ) + lockfile_packages |= develop | default else: develop = set( [canonicalize_name(k) for k in project.dev_packages.keys()] @@ -2103,24 +2099,40 @@ def do_uninstall( package_names = develop fix_venv_site(project.env_paths["lib"]) # Remove known "bad packages" from the list. + bad_pkgs = set([canonicalize_name(pkg) for pkg in BAD_PACKAGES]) for bad_package in BAD_PACKAGES: - if canonicalize_name(bad_package) in package_names: + normalized_bad_pkg = canonicalize_name(bad_package) + if normalized_bad_pkg in package_map: if environments.is_verbose(): click.echo("Ignoring {0}.".format(repr(bad_package)), err=True) - del package_names[package_names.index( - canonicalize_name(bad_package) - )] + pkg_name_index = package_names.index(package_map[normalized_bad_pkg]) + del package_names[pkg_name_index] used_packages = develop | default & installed_package_names failure = False packages_to_remove = set() + if all: + package_names = develop | default + click.echo( + crayons.normal(fix_utf8("Un-installing all packages from virtualenv…"), bold=True) + ) + do_purge(allow_global=system) + removed = package_names - bad_pkgs + project.remove_packages_from_pipfile(removed) + if lock: + do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) + return if all_dev: - packages_to_remove |= develop & installed_package_names - package_names = set([pkg_name for pkg_name in package_names]) + package_names = develop + else: + package_names = set([pkg_name for pkg_name in package_names]) + selected_pkg_map = { + canonicalize_name(p): p for p in package_names + } packages_to_remove = [ - pkg_name for pkg_name in packages - if canonicalize_name(pkg_name) in used_packages + p for normalized, p in selected_pkg_map.items() + if (normalized in used_packages and normalized not in bad_pkgs) ] - for package_name in package_names: + for normalized, package_name in selected_pkg_map.items(): click.echo( crayons.white( fix_utf8("Uninstalling {0}…".format(repr(package_name))), bold=True @@ -2142,7 +2154,9 @@ def do_uninstall( in_dev_packages = project.get_package_name_in_pipfile( package_name, dev=True ) - if not in_dev_packages and not in_packages: + if not (in_dev_packages or in_packages): + if normalized in lockfile_packages: + continue click.echo( "No package {0} to remove from Pipfile.".format( crayons.green(package_name) diff --git a/pipenv/project.py b/pipenv/project.py index 74debc24..33b8c106 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -934,7 +934,10 @@ class Project(object): has_comments_as_lines = False for section in ("dev-packages", "packages"): pipfile_section = self.parsed_pipfile.get(section, {}) - lines = [l for l in p[section].serialized().splitlines()] + lines = [ + l for l in parsed[section].serialized().splitlines() + if section in parsed.keys() + ] pipfile_packages = [ pkg_name for pkg_name in pipfile_section.keys() if pep423_name(pkg_name) in packages diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 28d90799..112791ab 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -430,12 +430,12 @@ class FileRequirement(BaseRequirement): @property def pipfile_part(self): - pipfile_dict = attr.asdict(self, filter=filter_none).copy() + excludes = ["_base_line", "_has_hashed_name", "setup_path"] + filter_func = lambda k, v: bool(v) is True and k.name not in excludes + pipfile_dict = attr.asdict(self, filter=filter_func).copy() name = pipfile_dict.pop("name") if "_uri_scheme" in pipfile_dict: pipfile_dict.pop("_uri_scheme") - if "setup_path" in pipfile_dict: - pipfile_dict.pop("setup_path") # For local paths and remote installable artifacts (zipfiles, etc) collision_keys = {"file", "uri", "path"} if self._uri_scheme: @@ -754,7 +754,7 @@ class VCSRequirement(FileRequirement): @property def pipfile_part(self): - excludes = ["_repo", "_base_line", "setup_path"] + excludes = ["_repo", "_base_line", "setup_path", "_has_hashed_name"] filter_func = lambda k, v: bool(v) is True and k.name not in excludes pipfile_dict = attr.asdict(self, filter=filter_func).copy() if "vcs" in pipfile_dict: From afc344258e0d0d9a093e51a9be40d47c4bb91eb1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 10:34:18 -0400 Subject: [PATCH 051/218] Include hashes from local artifacts - Fixes #2394 Signed-off-by: Dan Ryan --- news/2394.bugfix.rst | 1 + pipenv/utils.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 news/2394.bugfix.rst diff --git a/news/2394.bugfix.rst b/news/2394.bugfix.rst new file mode 100644 index 00000000..78868381 --- /dev/null +++ b/news/2394.bugfix.rst @@ -0,0 +1 @@ +Local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. diff --git a/pipenv/utils.py b/pipenv/utils.py index 33e930d6..ed91387d 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -229,6 +229,7 @@ def actually_resolve_deps( name = "PipCommand" constraints = [] + needs_hash = [] if not req_dir: req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") for dep in deps: @@ -240,6 +241,9 @@ def actually_resolve_deps( url = indexes[0] dep = " ".join(remainder) req = Requirement.from_line(dep) + new_ireq = req.as_ireq() + if getattr(new_ireq, "link", None) and new_ireq.link.is_wheel and new_ireq.link.scheme == 'file': + needs_hash.append(new_ireq) # extra_constraints = [] @@ -288,10 +292,13 @@ def actually_resolve_deps( constraints=constraints, repository=pypi, clear_caches=clear, prereleases=pre ) # pre-resolve instead of iterating to avoid asking pypi for hashes of editable packages - hashes = None + hashes = { + ireq: pypi._hash_cache.get_hash(ireq.link) + for ireq in constraints if getattr(ireq, "link", None) + and ireq.link.scheme == "file" and ireq.link.is_artifact + } try: results = resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) - hashes = resolver.resolve_hashes(results) resolved_tree.update(results) except (NoCandidateFound, DistributionNotFound, HTTPError) as e: click_echo( @@ -318,6 +325,11 @@ def actually_resolve_deps( ), err=True ) raise RuntimeError + else: + resolved_hashes = resolver.resolve_hashes(results) + for ireq, ireq_hashes in resolved_hashes.items(): + if ireq not in hashes: + hashes[ireq] = ireq_hashes return (resolved_tree, hashes, markers_lookup, resolver) From 740c589ef4be38604682faa95e074f7576a5334b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 10:44:18 -0400 Subject: [PATCH 052/218] Also prioritize remote non-pypi hashes if given - Fixes #3145 Signed-off-by: Dan Ryan --- news/2394.bugfix.rst | 2 +- news/3145.bugfix.rst | 1 + pipenv/utils.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 news/3145.bugfix.rst diff --git a/news/2394.bugfix.rst b/news/2394.bugfix.rst index 78868381..61ec9445 100644 --- a/news/2394.bugfix.rst +++ b/news/2394.bugfix.rst @@ -1 +1 @@ -Local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. +Remote non-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. diff --git a/news/3145.bugfix.rst b/news/3145.bugfix.rst new file mode 100644 index 00000000..61ec9445 --- /dev/null +++ b/news/3145.bugfix.rst @@ -0,0 +1 @@ +Remote non-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. diff --git a/pipenv/utils.py b/pipenv/utils.py index ed91387d..84c72979 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -295,7 +295,7 @@ def actually_resolve_deps( hashes = { ireq: pypi._hash_cache.get_hash(ireq.link) for ireq in constraints if getattr(ireq, "link", None) - and ireq.link.scheme == "file" and ireq.link.is_artifact + and ireq.link.is_artifact and not is_pypi_url(ireq.link.url) } try: results = resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) From b1d3aec5d2da9d011892131e7af425b2422d5d2f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 11:17:33 -0400 Subject: [PATCH 053/218] Don't write build artifacts to cwd - Fixes #3106 Signed-off-by: Dan Ryan --- news/3106.bugfix.rst | 1 + pipenv/utils.py | 6 +++++- pipenv/vendor/requirementslib/utils.py | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 news/3106.bugfix.rst diff --git a/news/3106.bugfix.rst b/news/3106.bugfix.rst new file mode 100644 index 00000000..70c0917f --- /dev/null +++ b/news/3106.bugfix.rst @@ -0,0 +1 @@ +Pipenv will avoid leaving build artifacts in the current working directory. diff --git a/pipenv/utils.py b/pipenv/utils.py index 84c72979..c2f2a9d8 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -295,7 +295,11 @@ def actually_resolve_deps( hashes = { ireq: pypi._hash_cache.get_hash(ireq.link) for ireq in constraints if getattr(ireq, "link", None) - and ireq.link.is_artifact and not is_pypi_url(ireq.link.url) + # We can only hash artifacts, but we don't want normal pypi artifcats since the + # normal resolver handles those + and ireq.link.is_artifact and not is_pypi_url(ireq.link.url) and not + # We also don't want to try to hash directories as this will fail (editable deps) + (ireq.link.scheme == "file" and os.path.isdir(ireq.link.path)) } try: results = resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index 75a05ce0..291c30ae 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -186,7 +186,14 @@ def ensure_setup_py(base_dir): if not base_dir: base_dir = create_tracked_tempdir(prefix="requirementslib-setup") base_dir = Path(base_dir) + if base_dir.exists() and base_dir.name == "setup.py": + base_dir = base_dir.parent + elif not (base_dir.exists() and base_dir.is_dir()): + base_dir = base_dir.parent + if not (base_dir.exists() and base_dir.is_dir()): + base_dir = base_dir.parent setup_py = base_dir.joinpath("setup.py") + is_new = False if setup_py.exists() else True if not setup_py.exists(): setup_py.write_text(u"") From 77b3edd3020feba434a7724ee1a224ff50342a1d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 11:35:20 -0400 Subject: [PATCH 054/218] Update requirementslib version Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/__init__.py | 2 +- pipenv/vendor/vendor.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index bcbff09e..5429d6ee 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.2.3' +__version__ = '1.2.4' import logging diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 2e00af0c..2fe5e47f 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -27,7 +27,7 @@ requests==2.20.0 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.2.3 +requirementslib==1.2.4 attrs==18.2.0 distlib==0.2.8 packaging==18.0 From 545a23662cad1b3d4e7351854093027b0619c8d0 Mon Sep 17 00:00:00 2001 From: Aliaksei Urbanski Date: Fri, 2 Nov 2018 20:40:04 +0300 Subject: [PATCH 055/218] Fix _get_virtualenv_hash function Case-sensitive filesystems were handled like case-insensitive and vice versa. Close #3151 This changes also: - Add PyCharm's config directory to .gitignore --- .gitignore | 3 +++ news/3151.bugfix | 1 + pipenv/project.py | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 news/3151.bugfix diff --git a/.gitignore b/.gitignore index 4809d065..766ffe3a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ .AppleDouble .LSOverride +# PyCharm +.idea/ + # VSCode .vscode diff --git a/news/3151.bugfix b/news/3151.bugfix new file mode 100644 index 00000000..96f14c10 --- /dev/null +++ b/news/3151.bugfix @@ -0,0 +1 @@ +Fix project path hashing logic in purpose to prevent collisions of virtual environments. diff --git a/pipenv/project.py b/pipenv/project.py index 195448d9..0b32811d 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -449,7 +449,7 @@ class Project(object): # In-project venv # "Proper" path casing (on non-case-sensitive filesystems). if ( - fnmatch.fnmatch("A", "a") + not fnmatch.fnmatch("A", "a") or self.is_venv_in_project() or get_workon_home().joinpath(venv_name).exists() ): From d5680fb61792cc3a7400e985936e3f82214bba89 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 17:57:50 -0400 Subject: [PATCH 056/218] Fix hash function Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 2a0743a3..61250f2b 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -65,7 +65,7 @@ class HashCache(SafeFileCache): if can_hash: # hash url WITH fragment hash_value = self.get(new_location.url) - if not hash_value: + if not hash_value and ((new_location.scheme == "file" and not os.path.isdir(new_location.path)) or new_location.scheme != "file"): hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None hash_value = hash_value.encode('utf8') if hash_value else None if can_hash: diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 3799ccf4..a96bf568 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -194,7 +194,7 @@ index bf69803..31b85b9 100644 + if can_hash: + # hash url WITH fragment + hash_value = self.get(new_location.url) -+ if not hash_value: ++ if not hash_value and ((new_location.scheme == "file" and not os.path.isdir(new_location.path)) or new_location.scheme != "file"): + hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None + hash_value = hash_value.encode('utf8') if hash_value else None + if can_hash: From 02b0de9f7bde9ffdd7de2dd95d31fd4ab0705aa6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 18:23:26 -0400 Subject: [PATCH 057/218] Revert "Fix hash function" This reverts commit d5680fb61792cc3a7400e985936e3f82214bba89. --- pipenv/patched/piptools/repositories/pypi.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 61250f2b..2a0743a3 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -65,7 +65,7 @@ class HashCache(SafeFileCache): if can_hash: # hash url WITH fragment hash_value = self.get(new_location.url) - if not hash_value and ((new_location.scheme == "file" and not os.path.isdir(new_location.path)) or new_location.scheme != "file"): + if not hash_value: hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None hash_value = hash_value.encode('utf8') if hash_value else None if can_hash: diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index a96bf568..3799ccf4 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -194,7 +194,7 @@ index bf69803..31b85b9 100644 + if can_hash: + # hash url WITH fragment + hash_value = self.get(new_location.url) -+ if not hash_value and ((new_location.scheme == "file" and not os.path.isdir(new_location.path)) or new_location.scheme != "file"): ++ if not hash_value: + hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None + hash_value = hash_value.encode('utf8') if hash_value else None + if can_hash: From 2632d0bd112882a4b640280c6045fcb303921a82 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 18:23:55 -0400 Subject: [PATCH 058/218] Fix url to path comparison Signed-off-by: Dan Ryan --- pipenv/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index c2f2a9d8..dd107db0 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -221,7 +221,10 @@ def actually_resolve_deps( from pipenv.patched.piptools import logging as piptools_logging from pipenv.patched.piptools.exceptions import NoCandidateFound from .vendor.requirementslib.models.requirements import Requirement - from .vendor.vistir.path import create_tracked_tempdir, create_tracked_tempfile + from .vendor.vistir.path import ( + create_tracked_tempdir, create_tracked_tempfile, url_to_path, + ) + from .vendor.vistir.compat import Path, to_native_string class PipCommand(Command): """Needed for pip-tools.""" @@ -294,12 +297,12 @@ def actually_resolve_deps( # pre-resolve instead of iterating to avoid asking pypi for hashes of editable packages hashes = { ireq: pypi._hash_cache.get_hash(ireq.link) - for ireq in constraints if getattr(ireq, "link", None) + for ireq in constraints if (getattr(ireq, "link", None) # We can only hash artifacts, but we don't want normal pypi artifcats since the # normal resolver handles those - and ireq.link.is_artifact and not is_pypi_url(ireq.link.url) and not + and ireq.link.is_artifact and not (is_pypi_url(ireq.link.url) or # We also don't want to try to hash directories as this will fail (editable deps) - (ireq.link.scheme == "file" and os.path.isdir(ireq.link.path)) + (ireq.link.scheme == "file" and Path(to_native_string(url_to_path(ireq.link.url))).is_dir()))) } try: results = resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) From 714ff657ca6d0888b932cdffc6fc768e7c9cb6ca Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 3 Nov 2018 00:29:19 -0400 Subject: [PATCH 059/218] Respect --skip-lock option for uninstall - Fixes #2848 Signed-off-by: Dan Ryan --- news/2848.bugfix.rst | 1 + pipenv/cli/command.py | 6 +++--- pipenv/cli/options.py | 1 + pipenv/core.py | 34 ++++++++++++++++++++++++++++------ 4 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 news/2848.bugfix.rst diff --git a/news/2848.bugfix.rst b/news/2848.bugfix.rst new file mode 100644 index 00000000..f88c030d --- /dev/null +++ b/news/2848.bugfix.rst @@ -0,0 +1 @@ +Updated ``pipenv uninstall`` to respect the ``--skip-lock`` argument. diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 40d3f0fe..74f66d7e 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -253,7 +253,7 @@ def install( @cli.command(short_help="Un-installs a provided package and removes it from Pipfile.") -@option("--lock", is_flag=True, default=True, help="Lock afterwards.") +@option("--skip-lock/--lock", is_flag=True, default=False, help="Lock afterwards.") @option( "--all-dev", is_flag=True, @@ -272,7 +272,7 @@ def install( def uninstall( ctx, state, - lock=False, + skip_lock=False, all_dev=False, all=False, **kwargs @@ -286,7 +286,7 @@ def uninstall( three=state.three, python=state.python, system=state.system, - lock=lock, + lock=not skip_lock, all_dev=all_dev, all=all, keep_outdated=state.installstate.keep_outdated, diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 99fc5344..dcdc1d9f 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -353,6 +353,7 @@ def uninstall_options(f): def lock_options(f): f = install_base_options(f) f = requirements_flag(f) + f = skip_lock_option(f) f = pre_option(f) return f diff --git a/pipenv/core.py b/pipenv/core.py index 85f9b2e9..894ecaa2 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -919,7 +919,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): pip_config = {} # Actually create the virtualenv. - nospin = os.environ.get("PIPENV_ACTIVE", environments.PIPENV_NOSPIN) + nospin = environments.PIPENV_NOSPIN c = vistir.misc.run(cmd, verbose=False, return_object=True, spinner_name=environments.PIPENV_SPINNER, combine_stderr=False, block=False, nospin=nospin, env=pip_config) @@ -934,7 +934,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): project_file_name = os.path.join(project.virtualenv_location, ".project") with open(project_file_name, "w") as f: f.write(vistir.misc.fs_str(project.project_directory)) - + fix_venv_site(project.env_paths["lib"]) # Say where the virtualenv is. do_where(virtualenv=True, bare=False) @@ -2039,6 +2039,7 @@ def do_uninstall( ): from .environments import PIPENV_USE_SYSTEM from .vendor.requirementslib.models.requirements import Requirement + from .vendor.requirementslib.models.lockfile import Lockfile from .vendor.packaging.utils import canonicalize_name # Automatically use an activated virtualenv. @@ -2117,9 +2118,17 @@ def do_uninstall( ) do_purge(allow_global=system) removed = package_names - bad_pkgs - project.remove_packages_from_pipfile(removed) - if lock: - do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) + if pipfile_remove: + project.remove_packages_from_pipfile(removed) + if lock: + do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) + else: + lockfile = project.get_or_create_lockfile() + for key in lockfile.default.keys(): + del lockfile.default[key] + for key in lockfile.develop.keys(): + del lockfile.develop[key] + lockfile.write() return if all_dev: package_names = develop @@ -2141,7 +2150,7 @@ def do_uninstall( # Uninstall the package. if package_name in packages_to_remove: cmd = "{0} uninstall {1} -y".format( - escape_grouped_arguments(which_pip()), package_name + escape_grouped_arguments(which_pip(allow_global=system)), package_name ) if environments.is_verbose(): click.echo("$ {0}".format(cmd)) @@ -2154,6 +2163,19 @@ def do_uninstall( in_dev_packages = project.get_package_name_in_pipfile( package_name, dev=True ) + if normalized in lockfile_packages: + click.echo("{0} {1} {2} {3}".format( + crayons.blue("Removing"), + crayons.green(package_name), + crayons.blue("from"), + crayons.white(fix_utf8("Pipfile.lock…"))) + ) + lockfile = project.get_or_create_lockfile() + if normalized in lockfile.default: + del lockfile.default[normalized] + if normalized in lockfile.develop: + del lockfile.develop[normalized] + lockfile.write() if not (in_dev_packages or in_packages): if normalized in lockfile_packages: continue From f9d950b8c68b242050e1cc11e30f34bbc0698792 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 3 Nov 2018 18:01:00 -0400 Subject: [PATCH 060/218] Fixed a bug with encountering removed pythons - Fixes #2983 Signed-off-by: Dan Ryan --- news/2983.bugfix.rst | 1 + pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/cli.py | 13 +++++++++++++ pipenv/vendor/pythonfinder/models/windows.py | 5 ++++- 4 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 news/2983.bugfix.rst diff --git a/news/2983.bugfix.rst b/news/2983.bugfix.rst new file mode 100644 index 00000000..acfe3780 --- /dev/null +++ b/news/2983.bugfix.rst @@ -0,0 +1 @@ +Pipenv will no longer fail when encountering python versions on Windows that were unintalled. diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 60192826..dbc0026b 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.6' +__version__ = '1.1.7.dev0' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index 6e7980fe..221cb2fd 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -56,7 +56,20 @@ def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsup click.secho("Searching for python: {0!s}".format(find.strip()), fg="yellow") found = finder.find_python_version(find.strip()) if found: + py = found.py_version + comes_from = getattr(py, "comes_from", None) + if comes_from is not None: + comes_from_path = getattr(comes_from, "path", found.path) + else: + comes_from_path = found.path + arch = getattr(py, "architecture", None) click.secho("Found python at the following locations:", fg="green") + click.secho( + "{py.name!s}: {py.version!s} ({py.architecture!s}) @ {comes_from!s}".format( + py=py, comes_from=comes_from_path + ), + fg="yellow", + ) sys.exit(0) else: click.secho("Failed to find matching executable...", fg="yellow") diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index e47bcc2c..f985630f 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -84,7 +84,10 @@ class WindowsFinder(BaseFinder): install_path = getattr(version_object.info, "install_path", None) if install_path is None: continue - path = ensure_path(install_path.__getattr__("")) + try: + path = ensure_path(install_path.__getattr__("")) + except AttributeError: + continue try: py_version = PythonVersion.from_windows_launcher(version_object) except InvalidPythonVersion: From a0dd0541aac6bffe944bc8849873748509465cd5 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 3 Nov 2018 18:34:45 -0400 Subject: [PATCH 061/218] Fix virtualenv path discovery - Fixes #3154 Signed-off-by: Dan Ryan --- news/3154.trivial.rst | 1 + pipenv/project.py | 46 ++++++++++++++----------------------------- 2 files changed, 16 insertions(+), 31 deletions(-) create mode 100644 news/3154.trivial.rst diff --git a/news/3154.trivial.rst b/news/3154.trivial.rst new file mode 100644 index 00000000..0c3277a6 --- /dev/null +++ b/news/3154.trivial.rst @@ -0,0 +1 @@ +Fixed an unreleased bug with determining project virtualenv paths across python versions. diff --git a/pipenv/project.py b/pipenv/project.py index 0b32811d..5c8568f0 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -667,17 +667,21 @@ class Project(object): def _get_editable_packages(self, dev=False): section = "dev-packages" if dev else "packages" + # section = "{0}-editable".format(section) packages = { k: v + # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() - if is_editable(v) + if is_editable(k) or is_editable(v) } return packages def _get_vcs_packages(self, dev=False): section = "dev-packages" if dev else "packages" + # section = "{0}-vcs".format(section) packages = { k: v + # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() if is_vcs(v) or is_vcs(k) } @@ -1132,42 +1136,22 @@ class Project(object): prefix = vistir.compat.Path(location) import importlib py_version = tuple([int(v) for v in self.py_version.split(".")]) + py_version_short = ".".join([str(v) for v in py_version[:2]]) + running_version = ".".join([str(v) for v in sys.version_info[:2]]) try: - with sys_version(py_version): - _virtualenv = importlib.import_module("virtualenv") - except ImportError: + _virtualenv = importlib.import_module("virtualenv") + except (ImportError, AttributeError): with vistir.contextmanagers.temp_path(): - from string import Formatter - formatter = Formatter() - import sysconfig - if getattr(sys, "real_prefix", None): - scheme = sysconfig._get_default_scheme() - sysconfig._INSTALL_SCHEMES["posix_prefix"]["purelib"] - if not scheme: - scheme = "posix_prefix" if not sys.platform == "win32" else "nt" - is_purelib = "purelib" in sysconfig._INSTALL_SCHEMES[scheme] - lib_key = "purelib" if is_purelib else "platlib" - lib = sysconfig._INSTALL_SCHEMES[scheme][lib_key] - fields = [field for _, field, _, _ in formatter.parse() if field] - config = { - "py_version_short": self._pyversion, - } - for field in fields: - if field not in config: - config[field] = prefix - sys.path = [ - os.path.join(sysconfig._INSTALL_SCHEMES[scheme][lib_key], "site-packages"), - ] + sys.path - with sys_version(py_version): - six.reload_module(importlib) - _virtualenv = importlib.import_module("virtualenv") + sys.path = vistir.misc.load_path(self.which("python")) + six.moves.reload_module(importlib) + _virtualenv = importlib.import_module("virtualenv") with sys_version(py_version): home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix()) paths = { - "lib": lib, - "include": inc, + "lib": lib.replace(running_version, py_version_short), + "include": inc.replace(running_version, py_version_short), "scripts": bin_, - "purelib": lib, + "purelib": lib.replace(running_version, py_version_short), "prefix": home, "base": home } From 0f5b365f84ba236156f47184641eaa13a8606024 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 3 Nov 2018 20:02:32 -0400 Subject: [PATCH 062/218] Performance fixes and updates - Cuts runtime of normal commands by about 2/3 Signed-off-by: Dan Ryan --- news/3158.feature.rst | 1 + pipenv/project.py | 46 ++-- .../vendor/requirementslib/models/__init__.py | 10 +- pipenv/vendor/requirementslib/models/cache.py | 10 +- .../requirementslib/models/dependencies.py | 59 +++-- .../requirementslib/models/requirements.py | 220 ++++++++++++------ .../requirementslib/models/resolvers.py | 5 +- pipenv/vendor/requirementslib/models/utils.py | 36 +-- pipenv/vendor/requirementslib/models/vcs.py | 8 +- pipenv/vendor/requirementslib/utils.py | 144 +++++++----- 10 files changed, 301 insertions(+), 238 deletions(-) create mode 100644 news/3158.feature.rst diff --git a/news/3158.feature.rst b/news/3158.feature.rst new file mode 100644 index 00000000..08113e54 --- /dev/null +++ b/news/3158.feature.rst @@ -0,0 +1 @@ +Improved runtime performance of no-op commands such as ``pipenv --venv`` by around 2/3. diff --git a/pipenv/project.py b/pipenv/project.py index 0b32811d..c03a0e83 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -6,7 +6,6 @@ import re import sys import glob import base64 -import itertools import fnmatch import hashlib import contoml @@ -47,7 +46,6 @@ from .environments import ( PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_CACHE_DIR ) -from requirementslib.utils import is_vcs def _normalized(p): @@ -173,6 +171,7 @@ class Project(object): def _build_package_list(self, package_section): """Returns a list of packages for pip-tools to consume.""" + from pipenv.vendor.requirementslib.utils import is_vcs ps = {} # TODO: Separate the logic for showing packages from the filters for supplying pip-tools for k, v in self.parsed_pipfile.get(package_section, {}).items(): @@ -670,11 +669,12 @@ class Project(object): packages = { k: v for k, v in self.parsed_pipfile.get(section, {}).items() - if is_editable(v) + if is_editable(k) or is_editable(v) } return packages def _get_vcs_packages(self, dev=False): + from pipenv.vendor.requirementslib.utils import is_vcs section = "dev-packages" if dev else "packages" packages = { k: v @@ -1132,42 +1132,22 @@ class Project(object): prefix = vistir.compat.Path(location) import importlib py_version = tuple([int(v) for v in self.py_version.split(".")]) + py_version_short = ".".join([str(v) for v in py_version[:2]]) + running_version = ".".join([str(v) for v in sys.version_info[:2]]) try: - with sys_version(py_version): - _virtualenv = importlib.import_module("virtualenv") - except ImportError: + _virtualenv = importlib.import_module("virtualenv") + except (ImportError, AttributeError): with vistir.contextmanagers.temp_path(): - from string import Formatter - formatter = Formatter() - import sysconfig - if getattr(sys, "real_prefix", None): - scheme = sysconfig._get_default_scheme() - sysconfig._INSTALL_SCHEMES["posix_prefix"]["purelib"] - if not scheme: - scheme = "posix_prefix" if not sys.platform == "win32" else "nt" - is_purelib = "purelib" in sysconfig._INSTALL_SCHEMES[scheme] - lib_key = "purelib" if is_purelib else "platlib" - lib = sysconfig._INSTALL_SCHEMES[scheme][lib_key] - fields = [field for _, field, _, _ in formatter.parse() if field] - config = { - "py_version_short": self._pyversion, - } - for field in fields: - if field not in config: - config[field] = prefix - sys.path = [ - os.path.join(sysconfig._INSTALL_SCHEMES[scheme][lib_key], "site-packages"), - ] + sys.path - with sys_version(py_version): - six.reload_module(importlib) - _virtualenv = importlib.import_module("virtualenv") + sys.path = vistir.misc.load_path(self.which("python")) + six.moves.reload_module(importlib) + _virtualenv = importlib.import_module("virtualenv") with sys_version(py_version): home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix()) paths = { - "lib": lib, - "include": inc, + "lib": lib.replace(running_version, py_version_short), + "include": inc.replace(running_version, py_version_short), "scripts": bin_, - "purelib": lib, + "purelib": lib.replace(running_version, py_version_short), "prefix": home, "base": home } diff --git a/pipenv/vendor/requirementslib/models/__init__.py b/pipenv/vendor/requirementslib/models/__init__.py index 99e519d6..e819cdd5 100644 --- a/pipenv/vendor/requirementslib/models/__init__.py +++ b/pipenv/vendor/requirementslib/models/__init__.py @@ -1,10 +1,2 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - - -__all__ = ["Requirement", "Lockfile", "Pipfile"] - - -from .requirements import Requirement -from .lockfile import Lockfile -from .pipfile import Pipfile +# This is intentionally left blank diff --git a/pipenv/vendor/requirementslib/models/cache.py b/pipenv/vendor/requirementslib/models/cache.py index 71701090..0b8c47b1 100644 --- a/pipenv/vendor/requirementslib/models/cache.py +++ b/pipenv/vendor/requirementslib/models/cache.py @@ -7,7 +7,6 @@ import json import os import sys -import requests import vistir from appdirs import user_cache_dir @@ -17,7 +16,6 @@ from packaging.requirements import Requirement from .utils import as_tuple, key_from_req, lookup_table, get_pinned_version from ..exceptions import FileExistsError -from ..utils import VCS_SUPPORT CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) @@ -195,16 +193,20 @@ class HashCache(SafeFileCache): avoid ssues where the location on the server changes. """ def __init__(self, *args, **kwargs): - session = kwargs.pop('session', requests.session()) + session = kwargs.pop("session", None) + if not session: + import requests + session = requests.session() cache_dir = kwargs.pop('cache_dir', CACHE_DIR) self.session = session kwargs.setdefault('directory', os.path.join(cache_dir, 'hash-cache')) super(HashCache, self).__init__(*args, **kwargs) def get_hash(self, location): + from pip_shims import VcsSupport # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs = VCS_SUPPORT + vcs = VcsSupport() orig_scheme = location.scheme new_location = copy.deepcopy(location) if orig_scheme in vcs.all_schemes: diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index c3df222f..48e84d0f 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -9,21 +9,17 @@ import attr import packaging.markers import packaging.version import requests -import warnings from first import first from packaging.utils import canonicalize_name -from pip_shims import ( - FormatControl, InstallRequirement, PackageFinder, RequirementPreparer, - RequirementSet, RequirementTracker, Resolver, WheelCache, pip_version -) +import pip_shims.shims from vistir.compat import JSONDecodeError, fs_str, ResourceWarning from vistir.contextmanagers import cd, temp_environ from vistir.misc import partialclass from vistir.path import create_tracked_tempdir -from ..utils import get_pip_command, prepare_pip_source_args, _ensure_dir +from ..utils import prepare_pip_source_args, _ensure_dir from .cache import CACHE_DIR, DependencyCache from .utils import ( clean_requires_python, fix_requires_python_marker, format_requirement, @@ -36,7 +32,9 @@ PKGS_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "pkgs")) WHEEL_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "wheels")) DEPENDENCY_CACHE = DependencyCache() -WHEEL_CACHE = WheelCache(CACHE_DIR, FormatControl(set(), set())) +WHEEL_CACHE = pip_shims.shims.WheelCache( + CACHE_DIR, pip_shims.shims.FormatControl(set(), set()) +) def _get_filtered_versions(ireq, versions, prereleases): @@ -65,6 +63,29 @@ def find_all_matches(finder, ireq, pre=False): return candidates +def get_pip_command(): + # Use pip's parser for pip.conf management and defaults. + # General options (find_links, index_url, extra_index_url, trusted_host, + # and pre) are defered to pip. + import optparse + + class PipCommand(pip_shims.shims.Command): + name = "PipCommand" + + pip_command = PipCommand() + pip_command.parser.add_option(pip_shims.shims.cmdoptions.no_binary()) + pip_command.parser.add_option(pip_shims.shims.cmdoptions.only_binary()) + index_opts = pip_shims.shims.cmdoptions.make_option_group( + pip_shims.shims.cmdoptions.index_group, pip_command.parser + ) + pip_command.parser.insert_option_group(0, index_opts) + pip_command.parser.add_option( + optparse.Option("--pre", action="store_true", default=False) + ) + + return pip_command + + @attr.s class AbstractDependency(object): name = attr.ib() @@ -242,7 +263,7 @@ def get_abstract_dependencies(reqs, sources=None, parent=None): from .requirements import Requirement for req in reqs: - if isinstance(req, InstallRequirement): + if isinstance(req, pip_shims.shims.InstallRequirement): requirement = Requirement.from_line( "{0}{1}".format(req.name, req.specifier) ) @@ -273,16 +294,16 @@ def get_dependencies(ireq, sources=None, parent=None): :return: A set of dependency lines for generating new InstallRequirements. :rtype: set(str) """ - if not isinstance(ireq, InstallRequirement): + if not isinstance(ireq, pip_shims.shims.InstallRequirement): name = getattr( ireq, "project_name", getattr(ireq, "project", ireq.name), ) version = getattr(ireq, "version", None) if not version: - ireq = InstallRequirement.from_line("{0}".format(name)) + ireq = pip_shims.shims.InstallRequirement.from_line("{0}".format(name)) else: - ireq = InstallRequirement.from_line("{0}=={1}".format(name, version)) + ireq = pip_shims.shims.InstallRequirement.from_line("{0}=={1}".format(name, version)) pip_options = get_pip_options(sources=sources) getters = [ get_dependencies_from_cache, @@ -354,7 +375,7 @@ def get_dependencies_from_json(ireq): if not requires_dist: # The API can return None for this. return for requires in requires_dist: - i = InstallRequirement.from_line(requires) + i = pip_shims.shims.InstallRequirement.from_line(requires) # See above, we don't handle requirements with extras. if not _marker_contains_extra(i): yield format_requirement(i) @@ -389,7 +410,7 @@ def get_dependencies_from_cache(ireq): try: broken = False for line in cached: - dep_ireq = InstallRequirement.from_line(line) + dep_ireq = pip_shims.shims.InstallRequirement.from_line(line) name = canonicalize_name(dep_ireq.name) if _marker_contains_extra(dep_ireq): broken = True # The "extra =" marker breaks everything. @@ -426,7 +447,7 @@ def get_dependencies_from_index(dep, sources=None, pip_options=None, wheel_cache if not wheel_cache: wheel_cache = WHEEL_CACHE dep.is_direct = True - reqset = RequirementSet() + reqset = pip_shims.shims.RequirementSet() reqset.add_requirement(dep) requirements = None setup_requires = {} @@ -554,7 +575,7 @@ def get_finder(sources=None, pip_command=None, pip_options=None): if not pip_options: pip_options = get_pip_options(sources=sources, pip_command=pip_command) session = pip_command._build_session(pip_options) - finder = PackageFinder( + finder = pip_shims.shims.PackageFinder( find_links=[], index_urls=[s.get("url") for s in sources], trusted_hosts=[], @@ -590,7 +611,7 @@ def start_resolver(finder=None, wheel_cache=None): _build_dir = create_tracked_tempdir(fs_str("build")) _source_dir = create_tracked_tempdir(fs_str("source")) preparer = partialclass( - RequirementPreparer, + pip_shims.shims.RequirementPreparer, build_dir=_build_dir, src_dir=_source_dir, download_dir=download_dir, @@ -599,7 +620,7 @@ def start_resolver(finder=None, wheel_cache=None): build_isolation=False, ) resolver = partialclass( - Resolver, + pip_shims.shims.Resolver, finder=finder, session=finder.session, upgrade_strategy="to-satisfy-only", @@ -612,8 +633,8 @@ def start_resolver(finder=None, wheel_cache=None): use_user_site=False, ) try: - if packaging.version.parse(pip_version) >= packaging.version.parse('18'): - with RequirementTracker() as req_tracker: + if packaging.version.parse(pip_shims.shims.pip_version) >= packaging.version.parse('18'): + with pip_shims.shims.RequirementTracker() as req_tracker: preparer = preparer(req_tracker=req_tracker) yield resolver(preparer=preparer) else: diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 112791ab..e1a1e6eb 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -9,35 +9,54 @@ import os from contextlib import contextmanager import attr -import six +import pip_shims from first import first from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement from packaging.specifiers import Specifier, SpecifierSet from packaging.utils import canonicalize_name -from pip_shims.shims import _strip_extras, parse_version, path_to_url, url_to_path, Link from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote from vistir.compat import FileNotFoundError, Path from vistir.misc import dedup from vistir.path import ( - create_tracked_tempdir, get_converted_relative_path, is_file_url, - is_valid_url + create_tracked_tempdir, + get_converted_relative_path, + is_file_url, + is_valid_url, ) from ..exceptions import RequirementError -from ..utils import VCS_LIST, is_installable_file, is_vcs, ensure_setup_py +from ..utils import ( + VCS_LIST, + is_installable_file, + is_vcs, + ensure_setup_py, + add_ssh_scheme_to_git_uri, + strip_ssh_from_git_uri, +) from .baserequirement import BaseRequirement -from .markers import PipenvMarkers from .utils import ( - HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string, - filter_none, format_requirement, get_version, init_requirement, - is_pinned_requirement, make_install_requirement, optional_instance_of, parse_extras, - specs_to_string, split_markers_from_line, ireq_from_editable, ireq_from_line, - split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs, normalize_name, create_link, - Requirement as PkgResourcesRequirement + HASH_STRING, + build_vcs_link, + extras_to_string, + filter_none, + format_requirement, + get_version, + init_requirement, + is_pinned_requirement, + make_install_requirement, + optional_instance_of, + parse_extras, + specs_to_string, + split_markers_from_line, + split_vcs_method_from_uri, + validate_path, + validate_specifiers, + validate_vcs, + normalize_name, + create_link, ) @@ -45,13 +64,15 @@ from .utils import ( class NamedRequirement(BaseRequirement): name = attr.ib() version = attr.ib(validator=attr.validators.optional(validate_specifiers)) - req = attr.ib(type=PkgResourcesRequirement) + req = attr.ib() extras = attr.ib(default=attr.Factory(list)) editable = attr.ib(default=False) @req.default def get_requirement(self): - req = init_requirement("{0}{1}".format(canonicalize_name(self.name), self.version)) + req = init_requirement( + "{0}{1}".format(canonicalize_name(self.name), self.version) + ) return req @classmethod @@ -84,7 +105,7 @@ class NamedRequirement(BaseRequirement): creation_args["version"] = version req = init_requirement("{0}{1}".format(name, version)) if extras: - req.extras += tuple(extras,) + req.extras += tuple(extras) creation_args["req"] = req return cls(**creation_args) @@ -187,9 +208,9 @@ class FileRequirement(BaseRequirement): parsed_url = urllib_parse.urlsplit(fixed_line) original_url = parsed_url._replace() - if added_ssh_scheme and ':' in parsed_url.netloc: - original_netloc, original_path_start = parsed_url.netloc.rsplit(':', 1) - uri_path = '/{0}{1}'.format(original_path_start, parsed_url.path) + if added_ssh_scheme and ":" in parsed_url.netloc: + original_netloc, original_path_start = parsed_url.netloc.rsplit(":", 1) + uri_path = "/{0}{1}".format(original_path_start, parsed_url.path) parsed_url = original_url._replace(netloc=original_netloc, path=uri_path) # Split the VCS part out if needed. @@ -205,12 +226,14 @@ class FileRequirement(BaseRequirement): if parsed_url.scheme == "file" and parsed_url.path: # This is a "file://" URI. Use url_to_path and path_to_url to # ensure the path is absolute. Also we need to build relpath. - path = Path(url_to_path(urllib_parse.urlunsplit(parsed_url))).as_posix() + path = Path( + pip_shims.shims.url_to_path(urllib_parse.urlunsplit(parsed_url)) + ).as_posix() try: relpath = get_converted_relative_path(path) except ValueError: relpath = None - uri = path_to_url(path) + uri = pip_shims.shims.path_to_url(path) else: # This is a remote URI. Simply use it. path = None @@ -221,7 +244,9 @@ class FileRequirement(BaseRequirement): ) if added_ssh_scheme: - original_uri = urllib_parse.urlunsplit(original_url._replace(scheme=original_scheme, fragment="")) + original_uri = urllib_parse.urlunsplit( + original_url._replace(scheme=original_scheme, fragment="") + ) uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. @@ -235,7 +260,7 @@ class FileRequirement(BaseRequirement): def get_uri(self): if self.path and not self.uri: self._uri_scheme = "path" - self.uri = path_to_url(os.path.abspath(self.path)) + self.uri = pip_shims.shims.path_to_url(os.path.abspath(self.path)) @name.default def get_name(self): @@ -247,6 +272,7 @@ class FileRequirement(BaseRequirement): return self.link.egg_fragment elif self.link and self.link.is_wheel: from pip_shims import Wheel + return Wheel(self.link.filename).name if ( self._uri_scheme != "uri" @@ -272,9 +298,9 @@ class FileRequirement(BaseRequirement): else: _path = self.path if self.editable: - _ireq = ireq_from_editable(_path.as_uri()) + _ireq = pip_shims.shims.ireq_from_editable(_path.as_uri()) else: - _ireq = ireq_from_line(_path.as_posix()) + _ireq = pip_shims.shims.ireq_from_line(_path.as_posix()) dist = make_abstract_dist(_ireq).get_dist() name = dist.project_name except (TypeError, ValueError, AttributeError) as e: @@ -361,6 +387,7 @@ class FileRequirement(BaseRequirement): } if link and link.is_wheel: from pip_shims import Wheel + arg_dict["name"] = Wheel(link.filename).name elif link.egg_fragment: arg_dict["name"] = link.egg_fragment @@ -399,7 +426,7 @@ class FileRequirement(BaseRequirement): uri_scheme = "file" if not uri: - uri = path_to_url(path) + uri = pip_shims.shims.path_to_url(path) link = create_link(uri) arg_dict = { @@ -414,12 +441,10 @@ class FileRequirement(BaseRequirement): @property def line_part(self): - if self._uri_scheme and self._uri_scheme == 'path': + if self._uri_scheme and self._uri_scheme == "path": seed = self.path or unquote(self.link.url_without_fragment) or self.uri - elif ( - (self._uri_scheme and self._uri_scheme == "file") - or ((self.link.is_artifact or self.link.is_wheel) - and self.link.url) + elif (self._uri_scheme and self._uri_scheme == "file") or ( + (self.link.is_artifact or self.link.is_wheel) and self.link.url ): seed = unquote(self.link.url_without_fragment) or self.uri # add egg fragments to remote artifacts (valid urls only) @@ -500,7 +525,7 @@ class VCSRequirement(FileRequirement): def __attrs_post_init__(self): if not self.uri: if self.path: - self.uri = path_to_url(self.path) + self.uri = pip_shims.shims.path_to_url(self.path) split = urllib_parse.urlsplit(self.uri) scheme, rest = split[0], split[1:] vcs_type = "" @@ -513,14 +538,14 @@ class VCSRequirement(FileRequirement): @link.default def get_link(self): - uri = self.uri if self.uri else path_to_url(self.path) + uri = self.uri if self.uri else pip_shims.shims.path_to_url(self.path) return build_vcs_link( self.vcs, add_ssh_scheme_to_git_uri(uri), name=self.name, ref=self.ref, subdirectory=self.subdirectory, - extras=self.extras + extras=self.extras, ) @name.default @@ -583,12 +608,12 @@ class VCSRequirement(FileRequirement): return self._repo def get_checkout_dir(self, src_dir=None): - src_dir = os.environ.get('PIP_SRC', None) if not src_dir else src_dir + src_dir = os.environ.get("PIP_SRC", None) if not src_dir else src_dir checkout_dir = None if self.is_local: path = self.path if not path: - path = url_to_path(self.uri) + path = pip_shims.shims.url_to_path(self.uri) if path and os.path.exists(path): checkout_dir = os.path.abspath(path) return checkout_dir @@ -596,6 +621,7 @@ class VCSRequirement(FileRequirement): def get_vcs_repo(self, src_dir=None): from .vcs import VCSRepository + checkout_dir = self.get_checkout_dir(src_dir=src_dir) link = build_vcs_link( self.vcs, @@ -603,7 +629,7 @@ class VCSRequirement(FileRequirement): name=self.name, ref=self.ref, subdirectory=self.subdirectory, - extras=self.extras + extras=self.extras, ) vcsrepo = VCSRepository( url=link.url, @@ -611,7 +637,7 @@ class VCSRequirement(FileRequirement): ref=self.ref if self.ref else None, checkout_directory=checkout_dir, vcs_type=self.vcs, - subdirectory=self.subdirectory + subdirectory=self.subdirectory, ) if not self.is_local: vcsrepo.obtain() @@ -661,7 +687,16 @@ class VCSRequirement(FileRequirement): creation_args = {} pipfile_keys = [ k - for k in ("ref", "vcs", "subdirectory", "path", "editable", "file", "uri", "extras") + for k in ( + "ref", + "vcs", + "subdirectory", + "path", + "editable", + "file", + "uri", + "extras", + ) + VCS_LIST if k in pipfile ] @@ -674,13 +709,20 @@ class VCSRequirement(FileRequirement): creation_args["vcs"] = key target = pipfile.get(key) drive, path = os.path.splitdrive(target) - if not drive and not os.path.exists(target) and (is_valid_url(target) or - is_file_url(target) or target.startswith('git@')): + if ( + not drive + and not os.path.exists(target) + and ( + is_valid_url(target) + or is_file_url(target) + or target.startswith("git@") + ) + ): creation_args["uri"] = target else: creation_args["path"] = target if os.path.isabs(target): - creation_args["uri"] = path_to_url(target) + creation_args["uri"] = pip_shims.shims.path_to_url(target) else: creation_args[key] = pipfile.get(key) creation_args["name"] = name @@ -694,7 +736,7 @@ class VCSRequirement(FileRequirement): line = line.split(" ", 1)[1] vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line(line) if not extras and link.egg_fragment: - name, extras = _strip_extras(link.egg_fragment) + name, extras = pip_shims.shims._strip_extras(link.egg_fragment) if extras: extras = parse_extras(extras) else: @@ -715,7 +757,7 @@ class VCSRequirement(FileRequirement): editable=editable, uri=uri, extras=extras, - base_line=line + base_line=line, ) @property @@ -725,7 +767,11 @@ class VCSRequirement(FileRequirement): base_link = self.link if not self.link: base_link = self.get_link() - final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) if base_link.egg_fragment else "{0}" + final_format = ( + "{{0}}#egg={0}".format(base_link.egg_fragment) + if base_link.egg_fragment + else "{0}" + ) base = final_format.format(self.vcs_uri) elif self._base_line: base = self._base_line @@ -759,7 +805,7 @@ class VCSRequirement(FileRequirement): pipfile_dict = attr.asdict(self, filter=filter_func).copy() if "vcs" in pipfile_dict: pipfile_dict = self._choose_vcs_source(pipfile_dict) - name, _ = _strip_extras(pipfile_dict.pop("name")) + name, _ = pip_shims.shims._strip_extras(pipfile_dict.pop("name")) return {name: pipfile_dict} @@ -806,7 +852,9 @@ class Requirement(object): @property def extras_as_pip(self): if self.extras: - return "[{0}]".format(",".join(sorted([extra.lower() for extra in self.extras]))) + return "[{0}]".format( + ",".join(sorted([extra.lower() for extra in self.extras])) + ) return "" @@ -847,6 +895,7 @@ class Requirement(object): @classmethod def from_line(cls, line): from pip_shims import InstallRequirement + if isinstance(line, InstallRequirement): line = format_requirement(line) hashes = None @@ -856,8 +905,8 @@ class Requirement(object): editable = line.startswith("-e ") line = line.split(" ", 1)[1] if editable else line line, markers = split_markers_from_line(line) - line, extras = _strip_extras(line) - specifiers = '' + line, extras = pip_shims.shims._strip_extras(line) + specifiers = "" if extras: extras = parse_extras(extras) line = line.strip('"').strip("'").strip() @@ -866,7 +915,9 @@ class Requirement(object): # Installable local files and installable non-vcs urls are handled # as files, generally speaking line_is_vcs = is_vcs(line) - if is_installable_file(line) or ((is_file_url(line) or is_valid_url(line)) and not line_is_vcs): + if is_installable_file(line) or ( + (is_file_url(line) or is_valid_url(line)) and not line_is_vcs + ): r = FileRequirement.from_line(line_with_prefix) elif line_is_vcs: r = VCSRequirement.from_line(line_with_prefix, extras=extras) @@ -886,7 +937,7 @@ class Requirement(object): version = line[spec_idx:] specifiers = version if not extras: - name, extras = _strip_extras(name) + name, extras = pip_shims.shims._strip_extras(name) if extras: extras = parse_extras(extras) if version: @@ -929,12 +980,14 @@ class Requirement(object): @classmethod def from_metadata(cls, name, version, extras, markers): - return cls.from_ireq(make_install_requirement( - name, version, extras=extras, markers=markers, - )) + return cls.from_ireq( + make_install_requirement(name, version, extras=extras, markers=markers) + ) @classmethod def from_pipfile(cls, name, pipfile): + from .markers import PipenvMarkers + _pipfile = {} if hasattr(pipfile, "keys"): _pipfile = dict(pipfile).copy() @@ -955,7 +1008,9 @@ class Requirement(object): r.req.marker = getattr(req_markers, "marker", None) r.req.specifier = SpecifierSet(_pipfile["version"]) extras = _pipfile.get("extras") - r.req.extras = sorted(dedup([extra.lower() for extra in extras])) if extras else [] + r.req.extras = ( + sorted(dedup([extra.lower() for extra in extras])) if extras else [] + ) args = { "name": r.name, "vcs": vcs, @@ -972,8 +1027,14 @@ class Requirement(object): cls_inst.req.req.line = cls_inst.as_line() return cls_inst - def as_line(self, sources=None, include_hashes=True, include_extras=True, - include_markers=True, as_list=False): + def as_line( + self, + sources=None, + include_hashes=True, + include_extras=True, + include_markers=True, + as_list=False, + ): """Format this requirement as a line in requirements.txt. If ``sources`` provided, it should be an sequence of mappings, containing @@ -1022,7 +1083,7 @@ class Requirement(object): def get_markers(self): markers = self.markers if markers: - fake_pkg = PackagingRequirement('fakepkg; {0}'.format(markers)) + fake_pkg = PackagingRequirement("fakepkg; {0}".format(markers)) markers = fake_pkg.markers return markers @@ -1030,15 +1091,15 @@ class Requirement(object): return Specifier(self.specifiers) def get_version(self): - return parse_version(self.get_specifier().version) + return pip_shims.shims.parse_version(self.get_specifier().version) def get_requirement(self): req_line = self.req.req.line - if req_line.startswith('-e '): + if req_line.startswith("-e "): _, req_line = req_line.split(" ", 1) req = init_requirement(self.name) req.line = req_line - req.specifier = SpecifierSet(self.specifiers if self.specifiers else '') + req.specifier = SpecifierSet(self.specifiers if self.specifiers else "") if self.is_vcs or self.is_file_or_url: req.url = self.req.link.url_without_fragment req.marker = self.get_markers() @@ -1064,8 +1125,8 @@ class Requirement(object): if k in good_keys } name = self.name - if 'markers' in req_dict and req_dict['markers']: - req_dict['markers'] = req_dict['markers'].replace('"', "'") + if "markers" in req_dict and req_dict["markers"]: + req_dict["markers"] = req_dict["markers"].replace('"', "'") base_dict = { k: v for k, v in self.req.pipfile_part[name].items() @@ -1094,11 +1155,11 @@ class Requirement(object): ireq_line = self.as_line(include_hashes=False) if self.editable or self.req.editable: if ireq_line.startswith("-e "): - ireq_line = ireq_line[len("-e "):] + ireq_line = ireq_line[len("-e ") :] with ensure_setup_py(self.req.setup_path): - ireq = ireq_from_editable(ireq_line) + ireq = pip_shims.shims.ireq_from_editable(ireq_line) else: - ireq = ireq_from_line(ireq_line) + ireq = pip_shims.shims.ireq_from_line(ireq_line) if not getattr(ireq, "req", None): ireq.req = self.req.req else: @@ -1127,12 +1188,11 @@ class Requirement(object): """ from .dependencies import get_dependencies + if not sources: - sources = [{ - 'name': 'pypi', - 'url': 'https://pypi.org/simple', - 'verify_ssl': True, - }] + sources = [ + {"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True} + ] return get_dependencies(self.as_ireq(), sources=sources) def get_abstract_dependencies(self, sources=None): @@ -1146,18 +1206,27 @@ class Requirement(object): :rtype: list[ :class:`~requirementslib.models.dependency.AbstractDependency` ] """ - from .dependencies import AbstractDependency, get_dependencies, get_abstract_dependencies + from .dependencies import ( + AbstractDependency, + get_dependencies, + get_abstract_dependencies, + ) + if not self.abstract_dep: - parent = getattr(self, 'parent', None) + parent = getattr(self, "parent", None) self.abstract_dep = AbstractDependency.from_requirement(self, parent=parent) if not sources: - sources = [{'url': 'https://pypi.org/simple', 'name': 'pypi', 'verify_ssl': True},] + sources = [ + {"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True} + ] if is_pinned_requirement(self.ireq): deps = self.get_dependencies() else: ireq = sorted(self.find_all_matches(), key=lambda k: k.version) deps = get_dependencies(ireq.pop(), sources=sources) - return get_abstract_dependencies(deps, sources=sources, parent=self.abstract_dep) + return get_abstract_dependencies( + deps, sources=sources, parent=self.abstract_dep + ) def find_all_matches(self, sources=None, finder=None): """Find all matching candidates for the current requirement. @@ -1171,6 +1240,7 @@ class Requirement(object): """ from .dependencies import get_finder, find_all_matches + if not finder: finder = get_finder(sources=sources) return find_all_matches(finder, self.as_ireq()) diff --git a/pipenv/vendor/requirementslib/models/resolvers.py b/pipenv/vendor/requirementslib/models/resolvers.py index 1a239390..bd773ba6 100644 --- a/pipenv/vendor/requirementslib/models/resolvers.py +++ b/pipenv/vendor/requirementslib/models/resolvers.py @@ -6,7 +6,6 @@ import six from pip_shims.shims import Wheel -from ..utils import log, VCS_SUPPORT from .cache import HashCache from .utils import format_requirement, is_pinned_requirement, version_from_ireq @@ -141,6 +140,7 @@ class DependencyResolver(object): # Coerce input into AbstractDependency instances. # We accept str, Requirement, and AbstractDependency as input. from .dependencies import AbstractDependency + from ..utils import log for dep in root_nodes: if isinstance(dep, six.string_types): dep = AbstractDependency.from_string(dep) @@ -193,7 +193,8 @@ class DependencyResolver(object): if ireq.editable: return set() - vcs = VCS_SUPPORT + from pip_shims import VcsSupport + vcs = VcsSupport() if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: return set() diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index d72542c4..fbaaf1a4 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -14,14 +14,10 @@ from attr import validators from first import first from packaging.markers import InvalidMarker, Marker, Op, Value, Variable from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet -from packaging.version import parse as parse_version -from packaging.requirements import Requirement as PackagingRequirement -from pkg_resources import Requirement - from vistir.misc import dedup -from ..utils import SCHEME_LIST, VCS_LIST, is_star +from ..utils import SCHEME_LIST, VCS_LIST, is_star, strip_ssh_from_git_uri, add_ssh_scheme_to_git_uri HASH_STRING = " --hash={0}" @@ -42,17 +38,8 @@ def create_link(link): return Link(link) -def ireq_from_line(ireq): - from pip_shims import InstallRequirement - return InstallRequirement.from_line(ireq) - - -def ireq_from_editable(ireq): - from pip_shims import InstallRequirement - return InstallRequirement.from_editable(ireq) - - def init_requirement(name): + from pkg_resources import Requirement req = Requirement.parse(name) req.vcs = None req.local_file = None @@ -74,6 +61,7 @@ def extras_to_string(extras): def parse_extras(extras_str): """Turn a string of extras into a parsed extras list""" + from pkg_resources import Requirement extras = Requirement.parse("fakepkg{0}".format(extras_to_string(extras_str))).extras return sorted(dedup([extra.lower() for extra in extras])) @@ -124,22 +112,6 @@ def get_version(pipfile_entry): return "" -def strip_ssh_from_git_uri(uri): - """Return git+ssh:// formatted URI to git+git@ format""" - if isinstance(uri, six.string_types): - uri = uri.replace("git+ssh://", "git+", 1) - return uri - - -def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pipenv.patched.notpip format""" - if isinstance(uri, six.string_types): - # Add scheme for parsing purposes, this is also what pip does - if uri.startswith("git+") and "://" not in uri: - uri = uri.replace("git+", "git+ssh://", 1) - return uri - - def split_markers_from_line(line): """Split markers from a dependency""" if not any(line.startswith(uri_prefix) for uri_prefix in SCHEME_LIST): @@ -483,6 +455,7 @@ def clean_requires_python(candidates): """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" all_candidates = [] sys_version = '.'.join(map(str, sys.version_info[:3])) + from packaging.version import parse as parse_version py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', sys_version)) for c in candidates: from_location = attrgetter("location.requires_python") @@ -504,6 +477,7 @@ def clean_requires_python(candidates): def fix_requires_python_marker(requires_python): + from packaging.requirements import Requirement as PackagingRequirement marker_str = '' if any(requires_python.startswith(op) for op in Specifier._operators.keys()): spec_dict = defaultdict(set) diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index 4efb9bd3..dd8cc3a4 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -1,11 +1,9 @@ # -*- coding=utf-8 -*- import attr -from pip_shims import VcsSupport, parse_version, pip_version import os +import pip_shims -VCS_SUPPORT = VcsSupport() - @attr.s class VCSRepository(object): @@ -20,6 +18,8 @@ class VCSRepository(object): @repo_instance.default def get_repo_instance(self): + from pip_shims import VcsSupport + VCS_SUPPORT = VcsSupport() backend = VCS_SUPPORT._registry.get(self.vcs_type) return backend(url=self.url) @@ -51,7 +51,7 @@ class VCSRepository(object): def update(self, ref): target_ref = self.repo_instance.make_rev_options(ref) - if parse_version(pip_version) > parse_version("18.0"): + if pip_shims.parse_version(pip_shims.pip_version) > pip_shims.parse_version("18.0"): self.repo_instance.update(self.checkout_directory, self.url, target_ref) else: self.repo_instance.update(self.checkout_directory, target_ref) diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index 291c30ae..312efbb3 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -16,22 +16,12 @@ six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) from six.moves import Mapping, Sequence, Set, ItemsView from six.moves.urllib.parse import urlparse, urlsplit -from pip_shims.shims import ( - Command, VcsSupport, cmdoptions, is_archive_file, - is_installable_dir as _is_installable_dir -) +import pip_shims.shims from vistir.compat import Path from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir VCS_LIST = ("git", "svn", "hg", "bzr") -VCS_SCHEMES = [] -SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") - -VCS_SUPPORT = VcsSupport() - -if not VCS_SCHEMES: - VCS_SCHEMES = VCS_SUPPORT.all_schemes def setup_logger(): @@ -47,8 +37,38 @@ def setup_logger(): log = setup_logger() +SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") + + +VCS_SCHEMES = [ + "git", + "git+http", + "git+https", + "git+ssh", + "git+git", + "git+file", + "hg", + "hg+http", + "hg+https", + "hg+ssh", + "hg+static-http", + "svn", + "svn+ssh", + "svn+http", + "svn+https", + "svn+svn", + "bzr", + "bzr+http", + "bzr+https", + "bzr+ssh", + "bzr+sftp", + "bzr+ftp", + "bzr+lp", +] + + def is_installable_dir(path): - if _is_installable_dir(path): + if pip_shims.shims.is_installable_dir(path): return True path = Path(path) pyproject = path.joinpath("pyproject.toml") @@ -60,15 +80,31 @@ def is_installable_dir(path): return False +def strip_ssh_from_git_uri(uri): + """Return git+ssh:// formatted URI to git+git@ format""" + if isinstance(uri, six.string_types): + uri = uri.replace("git+ssh://", "git+", 1) + return uri + + +def add_ssh_scheme_to_git_uri(uri): + """Cleans VCS uris from pip format""" + if isinstance(uri, six.string_types): + # Add scheme for parsing purposes, this is also what pip does + if uri.startswith("git+") and "://" not in uri: + uri = uri.replace("git+", "git+ssh://", 1) + return uri + + def is_vcs(pipfile_entry): """Determine if dictionary entry from Pipfile is for a vcs dependency.""" - if hasattr(pipfile_entry, "keys"): + if isinstance(pipfile_entry, Mapping): return any(key for key in pipfile_entry.keys() if key in VCS_LIST) elif isinstance(pipfile_entry, six.string_types): if not is_valid_url(pipfile_entry) and pipfile_entry.startswith("git+"): - from .models.utils import add_ssh_scheme_to_git_uri pipfile_entry = add_ssh_scheme_to_git_uri(pipfile_entry) + parsed_entry = urlsplit(pipfile_entry) return parsed_entry.scheme in VCS_SCHEMES return False @@ -88,7 +124,9 @@ def multi_split(s, split): def is_star(val): - return isinstance(val, six.string_types) and val == "*" + return (isinstance(val, six.string_types) and val == "*") or ( + isinstance(val, Mapping) and val.get("version", "") == "*" + ) def is_installable_file(path): @@ -125,7 +163,7 @@ def is_installable_file(path): if lookup_path.is_dir() and is_installable_dir(absolute_path): return True - elif lookup_path.is_file() and is_archive_file(absolute_path): + elif lookup_path.is_file() and pip_shims.shims.is_archive_file(absolute_path): return True return False @@ -139,9 +177,7 @@ def prepare_pip_source_args(sources, pip_args=None): pip_args.extend(["-i", sources[0]["url"]]) # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): - pip_args.extend( - ["--trusted-host", urlparse(sources[0]["url"]).hostname] - ) + pip_args.extend(["--trusted-host", urlparse(sources[0]["url"]).hostname]) # Add additional sources as extra indexes. if len(sources) > 1: for source in sources[1:]: @@ -154,28 +190,6 @@ def prepare_pip_source_args(sources, pip_args=None): return pip_args -class PipCommand(Command): - name = 'PipCommand' - - -def get_pip_command(): - # Use pip's parser for pip.conf management and defaults. - # General options (find_links, index_url, extra_index_url, trusted_host, - # and pre) are defered to pip. - import optparse - pip_command = PipCommand() - pip_command.parser.add_option(cmdoptions.no_binary()) - pip_command.parser.add_option(cmdoptions.only_binary()) - index_opts = cmdoptions.make_option_group( - cmdoptions.index_group, - pip_command.parser, - ) - pip_command.parser.insert_option_group(0, index_opts) - pip_command.parser.add_option(optparse.Option('--pre', action='store_true', default=False)) - - return pip_command - - @ensure_mkdir_p(mode=0o777) def _ensure_dir(path): return path @@ -204,7 +218,6 @@ def ensure_setup_py(base_dir): setup_py.unlink() - _UNSET = object() _REMAP_EXIT = object() @@ -251,6 +264,7 @@ class PathAccessError(KeyError, IndexError, TypeError): representing what can occur when looking up a path in a nested object. """ + def __init__(self, exc, seg, path): self.exc = exc self.seg = seg @@ -258,11 +272,14 @@ class PathAccessError(KeyError, IndexError, TypeError): def __repr__(self): cn = self.__class__.__name__ - return '%s(%r, %r, %r)' % (cn, self.exc, self.seg, self.path) + return "%s(%r, %r, %r)" % (cn, self.exc, self.seg, self.path) def __str__(self): - return ('could not access %r from path %r, got error: %r' - % (self.seg, self.path, self.exc)) + return "could not access %r from path %r, got error: %r" % ( + self.seg, + self.path, + self.exc, + ) def get_path(root, path, default=_UNSET): @@ -292,7 +309,7 @@ def get_path(root, path, default=_UNSET): ``PathAccessError`` exceptions be raised. """ if isinstance(path, six.string_types): - path = path.split('.') + path = path.split(".") cur = root try: for seg in path: @@ -308,8 +325,9 @@ def get_path(root, path, default=_UNSET): cur = cur[seg] except (ValueError, KeyError, IndexError, TypeError): if not getattr(cur, "__iter__", None): - exc = TypeError('%r object is not indexable' - % type(cur).__name__) + exc = TypeError( + "%r object is not indexable" % type(cur).__name__ + ) raise PathAccessError(exc, seg, path) except PathAccessError: if default is _UNSET: @@ -373,12 +391,13 @@ def dict_path_exit(path, key, old_parent, new_parent, new_items): except AttributeError: ret = new_parent.__class__(vals) # frozensets else: - raise RuntimeError('unexpected iterable type: %r' % type(new_parent)) + raise RuntimeError("unexpected iterable type: %r" % type(new_parent)) return ret -def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, - **kwargs): +def remap( + root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, **kwargs +): """The remap ("recursive map") function is used to traverse and transform nested structures. Lists, tuples, sets, and dictionaries are just a few of the data structures nested into heterogenous @@ -462,14 +481,14 @@ def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, # TODO: improve argument formatting in sphinx doc # TODO: enter() return (False, items) to continue traverse but cancel copy? if not callable(visit): - raise TypeError('visit expected callable, not: %r' % visit) + raise TypeError("visit expected callable, not: %r" % visit) if not callable(enter): - raise TypeError('enter expected callable, not: %r' % enter) + raise TypeError("enter expected callable, not: %r" % enter) if not callable(exit): - raise TypeError('exit expected callable, not: %r' % exit) - reraise_visit = kwargs.pop('reraise_visit', True) + raise TypeError("exit expected callable, not: %r" % exit) + reraise_visit = kwargs.pop("reraise_visit", True) if kwargs: - raise TypeError('unexpected keyword arguments: %r' % kwargs.keys()) + raise TypeError("unexpected keyword arguments: %r" % kwargs.keys()) path, registry, stack = (), {}, [(None, root)] new_items_stack = [] @@ -492,8 +511,10 @@ def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, new_parent, new_items = res except TypeError: # TODO: handle False? - raise TypeError('enter should return a tuple of (new_parent,' - ' items_iterator), not: %r' % res) + raise TypeError( + "enter should return a tuple of (new_parent," + " items_iterator), not: %r" % res + ) if new_items is not False: # traverse unless False is explicitly passed registry[id_value] = new_parent @@ -524,7 +545,7 @@ def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, try: new_items_stack[-1][1].append(visited_item) except IndexError: - raise TypeError('expected remappable root, not: %r' % root) + raise TypeError("expected remappable root, not: %r" % root) return value @@ -554,14 +575,15 @@ def merge_items(target_list, sourced=False): for t_name, target in target_list: if sourced: + def remerge_visit(path, key, value): source_map[path + (key,)] = t_name return True + else: remerge_visit = default_visit - ret = remap(target, enter=remerge_enter, visit=remerge_visit, - exit=remerge_exit) + ret = remap(target, enter=remerge_enter, visit=remerge_visit, exit=remerge_exit) if not sourced: return ret From c8cf4dd4cfd24b7831ec0b20025b13b25674d765 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 3 Nov 2018 21:58:25 -0400 Subject: [PATCH 063/218] Fix requirementslib synax error Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/requirements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index e1a1e6eb..28e0935c 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -298,9 +298,9 @@ class FileRequirement(BaseRequirement): else: _path = self.path if self.editable: - _ireq = pip_shims.shims.ireq_from_editable(_path.as_uri()) + _ireq = pip_shims.shims.install_req_from_editable(_path.as_uri()) else: - _ireq = pip_shims.shims.ireq_from_line(_path.as_posix()) + _ireq = pip_shims.shims.install_req_from_line(_path.as_posix()) dist = make_abstract_dist(_ireq).get_dist() name = dist.project_name except (TypeError, ValueError, AttributeError) as e: From 716f47f98830dd9ff2cc96c9dbd65d19af34c6e4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 3 Nov 2018 22:07:58 -0400 Subject: [PATCH 064/218] Fix missed pip_shims fix Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/requirements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 28e0935c..8d087d23 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1157,9 +1157,9 @@ class Requirement(object): if ireq_line.startswith("-e "): ireq_line = ireq_line[len("-e ") :] with ensure_setup_py(self.req.setup_path): - ireq = pip_shims.shims.ireq_from_editable(ireq_line) + ireq = pip_shims.shims.install_req_from_editable(ireq_line) else: - ireq = pip_shims.shims.ireq_from_line(ireq_line) + ireq = pip_shims.shims.install_req_from_line(ireq_line) if not getattr(ireq, "req", None): ireq.req = self.req.req else: From 563e14e562c5d27b22e5009c8c5b36bcb4573a67 Mon Sep 17 00:00:00 2001 From: jxltom Date: Sun, 4 Nov 2018 09:45:04 +0800 Subject: [PATCH 065/218] Fix can not concatenate str and type error during locking --- pipenv/vendor/delegator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py index cf6f91c8..367f52b0 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py @@ -112,7 +112,7 @@ class Command(object): if self.subprocess.before: result += self.subprocess.before - if self.subprocess.after and self.subprocess.after is not pexpect.EOF: + if self.subprocess.after and self.subprocess.after not in (pexpect.EOF, pexpect.TIMEOUT): try: result += self.subprocess.after except (pexpect.EOF, pexpect.TIMEOUT): From 888884653aa343a7029e55db4cc2a7905eea3c69 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 4 Nov 2018 01:20:26 -0500 Subject: [PATCH 066/218] Refactor Resolver into class and allow CI auto-cache-clear Signed-off-by: Dan Ryan --- pipenv/exceptions.py | 74 ++++++++-- pipenv/utils.py | 341 +++++++++++++++++++++++++++---------------- 2 files changed, 278 insertions(+), 137 deletions(-) diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 86643c8a..95feb859 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -84,7 +84,7 @@ class PipenvUsageError(UsageError): message = formatted_message.format(msg_prefix, crayons.white(message, bold=True)) self.message = message extra = kwargs.pop("extra", []) - UsageError.__init__(self, message, ctx) + UsageError.__init__(self, fix_utf8(message), ctx) self.extra = extra def show(self, file=None): @@ -98,8 +98,8 @@ class PipenvUsageError(UsageError): self.extra = [self.extra,] for extra in self.extra: if color: - extra = getattr(crayons, color, "normal") - click_echo(extra, file=file) + extra = getattr(crayons, color, "blue")(extra) + click_echo(fix_utf8(extra), file=file) hint = '' if (self.cmd is not None and self.cmd.get_help_option(self.ctx) is not None): @@ -123,7 +123,7 @@ class PipenvFileError(FileError): crayons.white("{0} not found!".format(filename), bold=True), message ) - FileError.__init__(self, filename=filename, hint=message, **kwargs) + FileError.__init__(self, filename=filename, hint=fix_utf8(message), **kwargs) self.extra = extra def show(self, file=None): @@ -133,20 +133,20 @@ class PipenvFileError(FileError): if isinstance(self.extra, six.string_types): self.extra = [self.extra,] for extra in self.extra: - click_echo(extra, file=file) + click_echo(fix_utf8(extra), file=file) click_echo(self.message, file=file) class PipfileNotFound(PipenvFileError): def __init__(self, filename="Pipfile", extra=None, **kwargs): extra = kwargs.pop("extra", []) - message = ("{0}. {1}".format( + message = ("{0} {1}".format( crayons.red("Aborting!", bold=True), crayons.white("Please ensure that the file exists and is located in your" " project root directory.", bold=True) ) ) - super(PipfileNotFound, self).__init__(filename, message=message, extra=extra, **kwargs) + super(PipfileNotFound, self).__init__(filename, message=fix_utf8(message), extra=extra, **kwargs) class LockfileNotFound(PipenvFileError): @@ -157,7 +157,7 @@ class LockfileNotFound(PipenvFileError): crayons.red("$ pipenv lock", bold=True), crayons.white("before you can continue.", bold=True) ) - super(LockfileNotFound, self).__init__(filename, message=message, extra=extra, **kwargs) + super(LockfileNotFound, self).__init__(filename, message=fix_utf8(message), extra=extra, **kwargs) class DeployException(PipenvUsageError): @@ -165,13 +165,13 @@ class DeployException(PipenvUsageError): if not message: message = crayons.normal("Aborting deploy", bold=True) extra = kwargs.pop("extra", []) - PipenvUsageError.__init__(message=message, extra=extra, **kwargs) + PipenvUsageError.__init__(message=fix_utf8(message), extra=extra, **kwargs) class PipenvOptionsError(PipenvUsageError): def __init__(self, option_name, message=None, ctx=None, **kwargs): extra = kwargs.pop("extra", []) - PipenvUsageError.__init__(self, message=message, ctx=ctx, **kwargs) + PipenvUsageError.__init__(self, message=fix_utf8(message), ctx=ctx, **kwargs) self.extra = extra self.option_name = option_name @@ -184,7 +184,7 @@ class PipfileException(PipenvFileError): hint = "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), hint) filename = project.pipfile_location extra = kwargs.pop("extra", []) - PipenvFileError.__init__(self, filename, hint, extra=extra, **kwargs) + PipenvFileError.__init__(self, filename, fix_utf8(hint), extra=extra, **kwargs) class SetupException(PipenvException): @@ -200,7 +200,7 @@ class VirtualenvException(PipenvException): "There was an unexpected error while activating your virtualenv. " "Continuing anyway..." ) - PipenvException.__init__(self, message, **kwargs) + PipenvException.__init__(self, fix_utf8(message), **kwargs) class VirtualenvActivationException(VirtualenvException): @@ -211,7 +211,7 @@ class VirtualenvActivationException(VirtualenvException): "not activated. Continuing anyway…" ) self.message = message - VirtualenvException.__init__(self, message, **kwargs) + VirtualenvException.__init__(self, fix_utf8(message), **kwargs) class VirtualenvCreationException(VirtualenvException): @@ -219,7 +219,7 @@ class VirtualenvCreationException(VirtualenvException): if not message: message = "Failed to create virtual environment." self.message = message - VirtualenvException.__init__(self, message, **kwargs) + VirtualenvException.__init__(self, fix_utf8(message), **kwargs) class UninstallError(PipenvException): @@ -235,5 +235,49 @@ class UninstallError(PipenvException): crayons.yellow(package, bold=True) ) self.exit_code = return_code - PipenvException.__init__(self, message=message, extra=extra) + PipenvException.__init__(self, message=fix_utf8(message), extra=extra) self.extra = extra + + +class CacheError(PipenvException): + def __init__(self, path, **kwargs): + message = "{0} {1} {2}\n{0}".format( + crayons.red("ERROR:", bold=True), + crayons.blue("Corrupt cache file"), + crayons.white(path), + crayons.white('Consider trying "pipenv lock --clear" to clear the cache.') + ) + super(PipenvException, self).__init__(message=fix_utf8(message)) + + +class ResolutionFailure(PipenvException): + def __init__(self, message, no_version_found=False): + extra = ( + "{0}: Your dependencies could not be resolved. You likely have a " + "mismatch in your sub-dependencies.\n " + "First try clearing your dependency cache with {1}, then try the original command again.\n " + "Alternatively, you can use {2} to bypass this mechanism, then run " + "{3} to inspect the situation.\n " + "Hint: try {4} if it is a pre-release dependency." + "".format( + crayons.red("Warning", bold=True), + crayons.red("$ pipenv lock --clear"), + crayons.red("$ pipenv install --skip-lock"), + crayons.red("$ pipenv graph"), + crayons.red("$ pipenv lock --pre"), + ), + ) + if "no version found at all" in message: + no_version_found = True + message = "{0} {1}".format( + crayons.red("ERROR:", bold=True), crayons.yellow(message) + ) + if no_version_found: + messsage = "{0}\n{1}".format( + message, + crayons.blue( + "Please check your version specifier and version number. " + "See PEP440 for more information." + ) + ) + super(ResolutionFailure, self).__init__(fix_utf8(message), extra=extra) diff --git a/pipenv/utils.py b/pipenv/utils.py index dd107db0..c0d5a356 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -201,6 +201,219 @@ def prepare_pip_source_args(sources, pip_args=None): return pip_args +def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): + from .vendor.requirementslib.models.requirements import Requirement + constraints = [] + for dep in deps: + if not dep: + continue + url = None + indexes, trusted_hosts, remainder = parse_indexes(dep) + if indexes: + url = indexes[0] + dep = " ".join(remainder) + req = Requirement.from_line(dep) + constraints.append(req.constraint_line) + + if url: + index_lookup[req.name] = project.get_source(url=url).get("name") + # strip the marker and re-add it later after resolution + # but we will need a fallback in case resolution fails + # eg pypiwin32 + if req.markers: + markers_lookup[req.name] = req.markers.replace('"', "'") + return constraints + + +class Resolver(object): + def __init__(self, constraints, req_dir, project, sources, clear=False, pre=False): + from pipenv.patched.piptools import logging as piptools_logging + if environments.is_verbose(): + logging.log.verbose = True + piptools_logging.log.verbose = True + self.initial_constraints = constraints + self.req_dir = req_dir + self.project = project + self.sources = sources + self.resolved_tree = set() + self.hashes = {} + self.clear = clear + self.pre = pre + self.results = None + self._pip_args = None + self._constraints = None + self._parsed_constraints = None + self._resolver = None + self._repository = None + self._session = None + self._constraint_file = None + self._pip_options = None + self._pip_command = None + self._retry_attempts = 0 + + def __repr__(self): + return ( + "".format(self=self) + ) + + def _get_pip_command(self): + from pip_shims.shims import Command + + class PipCommand(Command): + """Needed for pip-tools.""" + + name = "PipCommand" + + from pipenv.patched.piptools.scripts.compile import get_pip_command + return get_pip_command() + + @property + def pip_command(self): + if self._pip_command is None: + self._pip_command = self._get_pip_command() + return self._pip_command + + def prepare_pip_args(self): + pip_args = [] + if self.sources: + pip_args = prepare_pip_source_args(self.sources, pip_args) + return pip_args + + @property + def pip_args(self): + if self._pip_args is None: + self._pip_args = self.prepare_pip_args() + return self._pip_args + + def prepare_constraint_file(self): + from pipenv.vendor.vistir.path import create_tracked_tempfile + constraints_file = create_tracked_tempfile( + mode="w", + prefix="pipenv-", + suffix="-constraints.txt", + dir=self.req_dir, + delete=False, + ) + if self.sources: + requirementstxt_sources = " ".join(self.pip_args) if self.pip_args else "" + requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") + constraints_file.write(u"{0}\n".format(requirementstxt_sources)) + constraints = self.initial_constraints + constraints_file.write(u"\n".join([c for c in constraints])) + constraints_file.close() + return constraints_file.name + + @property + def constraint_file(self): + if self._constraint_file is None: + self._constraint_file = self.prepare_constraint_file() + return self._constraint_file + + @property + def pip_options(self): + if self._pip_options is None: + pip_options, _ = self.pip_command.parser.parse_args(self.pip_args) + pip_options.cache_dir = environments.PIPENV_CACHE_DIR + self._pip_options = pip_options + if environments.is_verbose(): + click_echo( + crayons.blue("Using pip: {0}".format(" ".join(self.pip_args))), err=True + ) + return self._pip_options + + @property + def session(self): + if self._session is None: + self._session = self.pip_command._build_session(self.pip_options) + return self._session + + @property + def repository(self): + if self._repository is None: + from pipenv.patched.piptools.repositories.pypi import PyPIRepository + self._repository = PyPIRepository( + pip_options=self.pip_options, use_json=False, session=self.session + ) + return self._repository + + @property + def constraints(self): + if self._constraints is None: + from pip_shims.shims import parse_requirements + self._constraints = parse_requirements( + self.constraint_file, finder=self.repository.finder, session=self.session, + options=self.pip_options + ) + return self._constraints + + @property + def parsed_constraints(self): + if self._parsed_constraints is None: + self._parsed_constraints = [c for c in self.constraints] + return self._parsed_constraints + + def get_resolver(self, clear=False, pre=False): + from pipenv.patched.piptools.resolver import Resolver + self._resolver = Resolver( + constraints=self.parsed_constraints, repository=self.repository, + clear_caches=clear, prereleases=pre, + ) + + def populate_file_hashes(self): + from pipenv.vendor.vistir.compat import Path, to_native_string + from pipenv.vendor.vistir.path import url_to_path + self.hashes.update({ + ireq: self.resolver._hash_cache.get_hash(ireq.link) + for ireq in self.parsed_constraints if (getattr(ireq, "link", None) + # We can only hash artifacts, but we don't want normal pypi artifcats since the + # normal resolver handles those + and ireq.link.is_artifact and not (is_pypi_url(ireq.link.url) or ( + # We also don't want to try to hash directories as this will fail + # as these are editable deps and are not hashable + ireq.link.scheme == "file" and + Path(to_native_string(url_to_path(ireq.link.url))).is_dir() + ))) + }) + + @property + def resolver(self): + if self._resolver is None: + self.get_resolver(clear=self.clear, pre=self.pre) + return self._resolver + + def resolve(self): + from pipenv.vendor.pip_shims.shims import DistributionNotFound + from pipenv.vendor.requests.exceptions import HTTPError + from pipenv.patched.piptools.exceptions import NoCandidateFound + from pipenv.patched.piptools.cache import CorruptCacheError + from .exceptions import CacheError, ResolutionFailure + try: + results = self.resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) + except CorruptCacheError as e: + if environments.PIPENV_IS_CI or self.clear: + if self._retry_attempts < 3: + self.get_resolver(clear=True, pre=self.pre) + self._retry_attempts += 1 + self.resolve() + else: + raise CacheError(e.path) + except (NoCandidateFound, DistributionNotFound, HTTPError) as e: + raise ResolutionFailure(message=str(e)) + else: + self.results = results + self.resolved_tree.update(results) + return self.resolved_tree + + def resolve_hashes(self): + if self.results is not None: + resolved_hashes = self.resolver.resolve_hashes(self.results) + for ireq, ireq_hashes in resolved_hashes.items(): + if ireq not in self.hashes: + self.hashes[ireq] = ireq_hashes + return self.hashes + + def actually_resolve_deps( deps, index_lookup, @@ -211,132 +424,16 @@ def actually_resolve_deps( pre, req_dir=None, ): - from .vendor.pip_shims.shims import ( - Command, parse_requirements, DistributionNotFound - ) - from .vendor.requests.exceptions import HTTPError - from pipenv.patched.piptools.resolver import Resolver - from pipenv.patched.piptools.repositories.pypi import PyPIRepository - from pipenv.patched.piptools.scripts.compile import get_pip_command - from pipenv.patched.piptools import logging as piptools_logging - from pipenv.patched.piptools.exceptions import NoCandidateFound - from .vendor.requirementslib.models.requirements import Requirement - from .vendor.vistir.path import ( - create_tracked_tempdir, create_tracked_tempfile, url_to_path, - ) - from .vendor.vistir.compat import Path, to_native_string + from pipenv.vendor.vistir.path import create_tracked_tempdir - class PipCommand(Command): - """Needed for pip-tools.""" - - name = "PipCommand" - - constraints = [] - needs_hash = [] if not req_dir: req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") - for dep in deps: - if not dep: - continue - url = None - indexes, trusted_hosts, remainder = parse_indexes(dep) - if indexes: - url = indexes[0] - dep = " ".join(remainder) - req = Requirement.from_line(dep) - new_ireq = req.as_ireq() - if getattr(new_ireq, "link", None) and new_ireq.link.is_wheel and new_ireq.link.scheme == 'file': - needs_hash.append(new_ireq) + constraints = get_resolver_metadata(deps, index_lookup, markers_lookup, project, + sources) + resolver = Resolver(constraints, req_dir, project, sources, clear=clear, pre=pre) + resolved_tree = resolver.resolve() + hashes = resolver.resolve_hashes() - # extra_constraints = [] - - if url: - index_lookup[req.name] = project.get_source(url=url).get("name") - # strip the marker and re-add it later after resolution - # but we will need a fallback in case resolution fails - # eg pypiwin32 - if req.markers: - markers_lookup[req.name] = req.markers.replace('"', "'") - constraints.append(req.constraint_line) - - pip_command = get_pip_command() - constraints_file = None - pip_args = [] - if sources: - pip_args = prepare_pip_source_args(sources, pip_args) - if environments.is_verbose(): - click_echo(crayons.blue("Using pip: {0}".format(" ".join(pip_args))), err=True) - constraints_file = create_tracked_tempfile( - mode="w", - prefix="pipenv-", - suffix="-constraints.txt", - dir=req_dir, - delete=False, - ) - if sources: - requirementstxt_sources = " ".join(pip_args) if pip_args else "" - requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") - constraints_file.write(u"{0}\n".format(requirementstxt_sources)) - constraints_file.write(u"\n".join([_constraint for _constraint in constraints])) - constraints_file.close() - pip_options, _ = pip_command.parser.parse_args(pip_args) - pip_options.cache_dir = environments.PIPENV_CACHE_DIR - session = pip_command._build_session(pip_options) - pypi = PyPIRepository(pip_options=pip_options, use_json=False, session=session) - constraints = parse_requirements( - constraints_file.name, finder=pypi.finder, session=pypi.session, options=pip_options - ) - constraints = [c for c in constraints] - if environments.is_verbose(): - logging.log.verbose = True - piptools_logging.log.verbose = True - resolved_tree = set() - resolver = Resolver( - constraints=constraints, repository=pypi, clear_caches=clear, prereleases=pre - ) - # pre-resolve instead of iterating to avoid asking pypi for hashes of editable packages - hashes = { - ireq: pypi._hash_cache.get_hash(ireq.link) - for ireq in constraints if (getattr(ireq, "link", None) - # We can only hash artifacts, but we don't want normal pypi artifcats since the - # normal resolver handles those - and ireq.link.is_artifact and not (is_pypi_url(ireq.link.url) or - # We also don't want to try to hash directories as this will fail (editable deps) - (ireq.link.scheme == "file" and Path(to_native_string(url_to_path(ireq.link.url))).is_dir()))) - } - try: - results = resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) - resolved_tree.update(results) - except (NoCandidateFound, DistributionNotFound, HTTPError) as e: - click_echo( - "{0}: Your dependencies could not be resolved. You likely have a " - "mismatch in your sub-dependencies.\n " - "First try clearing your dependency cache with {1}, then try the original command again.\n " - "Alternatively, you can use {2} to bypass this mechanism, then run " - "{3} to inspect the situation.\n " - "Hint: try {4} if it is a pre-release dependency." - "".format( - crayons.red("Warning", bold=True), - crayons.red("$ pipenv lock --clear"), - crayons.red("$ pipenv install --skip-lock"), - crayons.red("$ pipenv graph"), - crayons.red("$ pipenv lock --pre"), - ), - err=True, - ) - click_echo(crayons.blue(str(e)), err=True) - if "no version found at all" in str(e): - click_echo( - crayons.blue( - "Please check your version specifier and version number. See PEP440 for more information." - ), err=True - ) - raise RuntimeError - else: - resolved_hashes = resolver.resolve_hashes(results) - for ireq, ireq_hashes in resolved_hashes.items(): - if ireq not in hashes: - hashes[ireq] = ireq_hashes return (resolved_tree, hashes, markers_lookup, resolver) From 8d44604eb451e9a217a8f8ab64cdb1bfbe14bab4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 4 Nov 2018 01:52:27 -0500 Subject: [PATCH 067/218] Update delegator patch Signed-off-by: Dan Ryan --- .../vendoring/patches/vendor/delegator-close-filehandles.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch index 7e42237f..092a6994 100644 --- a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -17,7 +17,7 @@ index d15aeb97..cf6f91c8 100644 - if self.subprocess.after: - result += self.subprocess.after -+ if self.subprocess.after and self.subprocess.after is not pexpect.EOF: ++ if self.subprocess.after and self.subprocess.after not in (pexpect.EOF, pexpect.TIMEOUT): + try: + result += self.subprocess.after + except (pexpect.EOF, pexpect.TIMEOUT): From cce60cc5f14ff95636f996bfa7dcbc0d14efa137 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 4 Nov 2018 02:19:43 -0500 Subject: [PATCH 068/218] Update vendored versions of pythonfinder and requirementslib Signed-off-by: Dan Ryan --- pipenv/vendor/vendor.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 2fe5e47f..45ff0384 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -21,13 +21,13 @@ pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.6 +pythonfinder==1.1.7 requests==2.20.0 chardet==3.0.4 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.2.4 +requirementslib==1.2.5 attrs==18.2.0 distlib==0.2.8 packaging==18.0 From 3977307bcb3e2499b94acc2a6e60b433592e7ab4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 4 Nov 2018 02:29:21 -0500 Subject: [PATCH 069/218] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/models/path.py | 5 +++++ pipenv/vendor/pythonfinder/models/pyenv.py | 17 ++++++++++++++--- pipenv/vendor/requirementslib/__init__.py | 2 +- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index dbc0026b..0b22546a 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.7.dev0' +__version__ = '1.1.7' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 20b2c196..33b4ab58 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -144,7 +144,12 @@ class SystemPath(object): self.path_order = ( before_path + [p.as_posix() for p in root_paths] + after_path ) + pyenv_shim_path = os.path.join(PYENV_ROOT, "shims") + if pyenv_shim_path in self.path_order: + self.path_order.remove(pyenv_shim_path) self.paths.update(self.pyenv_finder.roots) + if pyenv_shim_path in self.paths: + del self.paths[pyenv_shim_path] self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 1595a963..5874af64 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -44,6 +44,19 @@ class PyenvFinder(BaseFinder, BasePath): if path is not None ) + def get_version_order(self): + version_order_file = self.root.joinpath("version").read_text(encoding="utf-8") + version_paths = [ + p for p in self.root.glob("versions/*") + if not (p.parent.name == "envs" or p.name == "envs") + ] + versions = {v.name: v for v in version_paths} + version_order = [versions[v] for v in version_order_file.splitlines() if v in versions] + for version in version_order: + version_paths.remove(version) + version_order += version_paths + return version_order + @classmethod def version_from_bin_dir(cls, base_dir, name=None): py_version = None @@ -59,9 +72,7 @@ class PyenvFinder(BaseFinder, BasePath): def get_versions(self): versions = defaultdict() bin_ = sysconfig._INSTALL_SCHEMES[sysconfig._get_default_scheme()]["scripts"] - for p in self.root.glob("versions/*"): - if p.parent.name == "envs" or p.name == "envs": - continue + for p in self.get_version_order(): bin_dir = Path(bin_.format(base=p.as_posix())) version_path = None if bin_dir.exists(): diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 5429d6ee..ba0ce9ae 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.2.4' +__version__ = '1.2.5' import logging From 9c715289e9d132d5e6c37d82539067b27f8101a1 Mon Sep 17 00:00:00 2001 From: jxltom Date: Mon, 5 Nov 2018 20:20:13 +0800 Subject: [PATCH 070/218] Fix typo on spinner result text --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index bf4145e5..2091bd08 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -323,7 +323,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): except Exception: sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Failed...")) else: - sp.ok(environments.PIPENV_SPINNER_FAIL_TEXT.format("Success!")) + sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) # Warn the user of side-effects. click.echo( u"{0}: Your {1} now contains pinned versions, if your {2} did. \n" From d04b6b211d719da44a2f26c32a261e478c514325 Mon Sep 17 00:00:00 2001 From: David Staheli Date: Mon, 5 Nov 2018 10:13:51 -0500 Subject: [PATCH 071/218] Update YAML syntax --- .azure-pipelines/docs.yml | 21 ++++++++ .azure-pipelines/jobs/run-manifest-check.yml | 14 ++++++ .azure-pipelines/jobs/run-tests-windows.yml | 12 +++++ .azure-pipelines/jobs/run-tests.yml | 37 ++++++++++++++ .azure-pipelines/jobs/run-vendor-scripts.yml | 39 +++++++++++++++ .azure-pipelines/jobs/test.yml | 48 +++++++++++++++++++ .azure-pipelines/linux.yml | 27 +++++++++++ .azure-pipelines/steps/create-virtualenv.yml | 6 +++ .../steps/install-dependencies.yml | 3 ++ .azure-pipelines/steps/run-tests.yml | 23 +++++++++ .azure-pipelines/windows.yml | 23 +++++++++ 11 files changed, 253 insertions(+) create mode 100644 .azure-pipelines/docs.yml create mode 100644 .azure-pipelines/jobs/run-manifest-check.yml create mode 100644 .azure-pipelines/jobs/run-tests-windows.yml create mode 100644 .azure-pipelines/jobs/run-tests.yml create mode 100644 .azure-pipelines/jobs/run-vendor-scripts.yml create mode 100644 .azure-pipelines/jobs/test.yml create mode 100644 .azure-pipelines/linux.yml create mode 100644 .azure-pipelines/steps/create-virtualenv.yml create mode 100644 .azure-pipelines/steps/install-dependencies.yml create mode 100644 .azure-pipelines/steps/run-tests.yml create mode 100644 .azure-pipelines/windows.yml diff --git a/.azure-pipelines/docs.yml b/.azure-pipelines/docs.yml new file mode 100644 index 00000000..b65fdfb4 --- /dev/null +++ b/.azure-pipelines/docs.yml @@ -0,0 +1,21 @@ +jobs: +- job: + displayName: Docs + pool: + vmImage: ubuntu-16.04 + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + + - template: steps/install-dependencies.yml + + - bash: tox -e docs + displayName: Build docs + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: docs' + inputs: + pathToPublish: docs/build + artifactName: docs diff --git a/.azure-pipelines/jobs/run-manifest-check.yml b/.azure-pipelines/jobs/run-manifest-check.yml new file mode 100644 index 00000000..6d0b97eb --- /dev/null +++ b/.azure-pipelines/jobs/run-manifest-check.yml @@ -0,0 +1,14 @@ +steps: +- task: UsePythonVersion@0 + displayName: Use Python $(python.version) + inputs: + versionSpec: '$(python.version)' + architecture: '$(python.architecture)' + +- template: ../steps/install-dependencies.yml + +- bash: | + export GIT_SSL_CAINFO=$(python -m certifi) + export LANG=C.UTF-8 + python -m pip install check-manifest + check-manifest diff --git a/.azure-pipelines/jobs/run-tests-windows.yml b/.azure-pipelines/jobs/run-tests-windows.yml new file mode 100644 index 00000000..6b6f86fa --- /dev/null +++ b/.azure-pipelines/jobs/run-tests-windows.yml @@ -0,0 +1,12 @@ +steps: +- task: UsePythonVersion@0 + displayName: Use Python $(python.version) + inputs: + versionSpec: '$(python.version)' + architecture: '$(python.architecture)' + +- template: ../steps/install-dependencies.yml + +- template: ../steps/create-virtualenv.yml + +- template: ../steps/run-tests.yml diff --git a/.azure-pipelines/jobs/run-tests.yml b/.azure-pipelines/jobs/run-tests.yml new file mode 100644 index 00000000..fe98e8ec --- /dev/null +++ b/.azure-pipelines/jobs/run-tests.yml @@ -0,0 +1,37 @@ +steps: +- task: UsePythonVersion@0 + displayName: Use Python $(python.version) + inputs: + versionSpec: '$(python.version)' + architecture: '$(python.architecture)' + +- template: ../steps/install-dependencies.yml + +- bash: | + mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" + mkdir -p "$WORKON_HOME" + pip install certifi + export GIT_SSL_CAINFO="$(python -m certifi)" + export LANG="C.UTF-8" + export PIP_PROCESS_DEPENDENCY_LINKS="1" + echo "Path $PATH" + echo "Installing Pipenv…" + pip install -e "$(pwd)" --upgrade + pipenv install --deploy --dev + echo pipenv --venv && echo pipenv --py && echo pipenv run python --version + displayName: Make Virtualenv + +- script: | + # Fix Git SSL errors + export GIT_SSL_CAINFO="$(python -m certifi)" + export LANG="C.UTF-8" + export PIP_PROCESS_DEPENDENCY_LINKS="1" + pipenv run pytest --junitxml=test-results.xml + displayName: Run integration tests + +- task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFiles: '**/test-results.xml' + testRunTitle: 'Python $(python.version)' + condition: succeededOrFailed() diff --git a/.azure-pipelines/jobs/run-vendor-scripts.yml b/.azure-pipelines/jobs/run-vendor-scripts.yml new file mode 100644 index 00000000..7fe75731 --- /dev/null +++ b/.azure-pipelines/jobs/run-vendor-scripts.yml @@ -0,0 +1,39 @@ +parameters: + vmImage: + +jobs: +- job: Vendor_Scripts + displayName: Test Vendor Scripts + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 4 + matrix: + ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: + # TODO remove once vs2017-win2016 has Python 3.7 + Python37: + python.version: '>= 3.7.0-b2' + python.architecture: x64 + ${{ if ne(parameters.vmImage, 'vs2017-win2016' )}}: + Python37: + python.version: '>= 3.7' + python.architecture: x64 + steps: + - task: UsePythonVersion@0 + displayName: Use Python $(python.version) + inputs: + versionSpec: '$(python.version)' + architecture: '$(python.architecture)' + + - template: ../steps/install-dependencies.yml + + - bash: | + mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" + mkdir -p "$WORKON_HOME" + pip install certifi + export GIT_SSL_CAINFO=$(python -m certifi) + export LANG=C.UTF-8 + python -m pip install --upgrade invoke requests parver + python -m invoke vendoring.update + + - template: ./run-manifest-check.yml diff --git a/.azure-pipelines/jobs/test.yml b/.azure-pipelines/jobs/test.yml new file mode 100644 index 00000000..150937ef --- /dev/null +++ b/.azure-pipelines/jobs/test.yml @@ -0,0 +1,48 @@ +parameters: + vmImage: + +jobs: +- job: Test_Primary + displayName: Test Primary + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 4 + matrix: + Python27: + python.version: '2.7' + python.architecture: x64 + ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: + # TODO remove once vs2017-win2016 has Python 3.7 + Python37: + python.version: '>= 3.7.0-b2' + python.architecture: x64 + ${{ if ne(parameters.vmImage, 'vs2017-win2016' )}}: + Python37: + python.version: '>= 3.7' + python.architecture: x64 + steps: + - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: + - template: ./run-tests-windows.yml + + - ${{ if ne(parameters.vmImage, 'vs2017-win2016') }}: + - template: ./run-tests.yml + +- job: Test_Secondary + displayName: Test python3.6 + # Run after Test_Primary so we don't devour time and jobs if tests are going to fail + # dependsOn: Test_Primary + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 4 + matrix: + Python36: + python.version: '3.6' + python.architecture: x64 + steps: + - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: + - template: ./run-tests-windows.yml + + - ${{ if ne(parameters.vmImage, 'vs2017-win2016') }}: + - template: ./run-tests.yml diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml new file mode 100644 index 00000000..a2f34d4c --- /dev/null +++ b/.azure-pipelines/linux.yml @@ -0,0 +1,27 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + +jobs: +- template: jobs/test.yml + parameters: + vmImage: ubuntu-16.04 + +- template: jobs/run-vendor-scripts.yml + parameters: + vmImage: ubuntu-16.04 diff --git a/.azure-pipelines/steps/create-virtualenv.yml b/.azure-pipelines/steps/create-virtualenv.yml new file mode 100644 index 00000000..9d1a6903 --- /dev/null +++ b/.azure-pipelines/steps/create-virtualenv.yml @@ -0,0 +1,6 @@ +steps: +- script: | + virtualenv D:\.venv + D:\.venv\Scripts\pip.exe install -e . && D:\.venv\Scripts\pipenv install --dev + echo D:\.venv\Scripts\pipenv --venv && echo D:\.venv\Scripts\pipenv --py && echo D:\.venv\Scripts\pipenv run python --version + displayName: Make Virtualenv diff --git a/.azure-pipelines/steps/install-dependencies.yml b/.azure-pipelines/steps/install-dependencies.yml new file mode 100644 index 00000000..dfe733b6 --- /dev/null +++ b/.azure-pipelines/steps/install-dependencies.yml @@ -0,0 +1,3 @@ +steps: +- script: 'python -m pip install --upgrade pip && python -m pip install -e .' + displayName: Upgrade Pip & Install Pipenv diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml new file mode 100644 index 00000000..a7b99a13 --- /dev/null +++ b/.azure-pipelines/steps/run-tests.yml @@ -0,0 +1,23 @@ +steps: +- powershell: | + # Fix Git SSL errors + pip install certifi + python -m certifi > cacert.txt + Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" + $env:GIT_SSL_CAINFO="$(Get-Content cacert.txt)" + # Shorten paths to get under MAX_PATH or else integration tests will fail + # https://bugs.python.org/issue18199 + subst T: "$env:TEMP" + Write-Host "##vso[task.setvariable variable=TEMP]T:\" + $env:TEMP='T:\' + Write-Host "##vso[task.setvariable variable=TMP]T:\" + $env:TEMP='T:\' + D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests + displayName: Run integration tests + +- task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFiles: '**/test-results.xml' + testRunTitle: 'Python $(python.version)' + condition: succeededOrFailed() diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml new file mode 100644 index 00000000..35d5c16e --- /dev/null +++ b/.azure-pipelines/windows.yml @@ -0,0 +1,23 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + +jobs: +- template: jobs/test.yml + parameters: + vmImage: vs2017-win2016 From 374ac589bd689748640303c41cd774d28700e177 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 5 Nov 2018 11:22:08 -0500 Subject: [PATCH 072/218] Fix PIP_SHIMs import, pythonfinder, vistir - Add new exceptions - Improve validation - Cleanup spinner text - Fix pyenv finder when `PYENV_ROOT` is not set - Fix import errors for users installing and using `PIP_SHIMS` in pipenv Signed-off-by: Dan Ryan --- pipenv/cli/options.py | 26 +++++++------- pipenv/core.py | 40 ++++++++++++---------- pipenv/exceptions.py | 25 +++++++++++++- pipenv/vendor/pythonfinder/environment.py | 4 ++- pipenv/vendor/pythonfinder/models/pyenv.py | 2 +- pipenv/vendor/vistir/__init__.py | 2 +- pipenv/vendor/vistir/spin.py | 4 +++ 7 files changed, 68 insertions(+), 35 deletions(-) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index dcdc1d9f..b701ef90 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import os -from click import BOOL as click_booltype +import click.types from click import ( BadParameter, Group, Option, argument, echo, make_pass_decorator, option ) @@ -103,7 +103,7 @@ def extra_index_option(f): def editable_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) - state.installstate.editables.extend(value) + state.installstate.editables.extend(validate_requirements(ctx, param, value)) return value return option('-e', '--editable', expose_value=False, multiple=True, help='An editable python package URL or path, often to a VCS repo.', @@ -117,7 +117,7 @@ def sequential_option(f): return value return option("--sequential", is_flag=True, default=False, expose_value=False, help="Install dependencies one-at-a-time, instead of concurrently.", - callback=callback, type=click_booltype)(f) + callback=callback, type=click.types.BOOL)(f) def skip_lock_option(f): @@ -127,7 +127,7 @@ def skip_lock_option(f): return value return option("--skip-lock", is_flag=True, default=False, expose_value=False, help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", - callback=callback, type=click_booltype)(f) + callback=callback, type=click.types.BOOL)(f) def keep_outdated_option(f): @@ -137,7 +137,7 @@ def keep_outdated_option(f): return value return option("--keep-outdated", is_flag=True, default=False, expose_value=False, help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", - callback=callback, type=click_booltype)(f) + callback=callback, type=click.types.BOOL)(f) def selective_upgrade_option(f): @@ -145,7 +145,7 @@ def selective_upgrade_option(f): state = ctx.ensure_object(State) state.installstate.selective_upgrade = value return value - return option("--selective-upgrade", is_flag=True, default=False, type=click_booltype, + return option("--selective-upgrade", is_flag=True, default=False, type=click.types.BOOL, help="Update specified packages.", callback=callback, expose_value=False)(f) @@ -165,7 +165,7 @@ def dev_option(f): state = ctx.ensure_object(State) state.installstate.dev = value return value - return option("--dev", "-d", is_flag=True, default=False, type=click_booltype, + return option("--dev", "-d", is_flag=True, default=False, type=click.types.BOOL, help="Install both develop and default packages.", callback=callback, expose_value=False)(f) @@ -176,13 +176,13 @@ def pre_option(f): state.installstate.pre = value return value return option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.", - callback=callback, type=click_booltype, expose_value=False)(f) + callback=callback, type=click.types.BOOL, expose_value=False)(f) def package_arg(f): def callback(ctx, param, value): state = ctx.ensure_object(State) - state.installstate.packages.extend(value) + state.installstate.packages.extend(validate_requirements(ctx, param, value)) return value return argument('packages', nargs=-1, callback=callback, expose_value=False,)(f) @@ -235,7 +235,7 @@ def site_packages_option(f): state = ctx.ensure_object(State) state.site_packages = value return value - return option("--site-packages", is_flag=True, default=False, type=click_booltype, + return option("--site-packages", is_flag=True, default=False, type=click.types.BOOL, help="Enable site-packages for the virtualenv.", callback=callback, expose_value=False)(f) @@ -245,7 +245,7 @@ def clear_option(f): state = ctx.ensure_object(State) state.clear = value return value - return option("--clear", is_flag=True, callback=callback, type=click_booltype, + return option("--clear", is_flag=True, callback=callback, type=click.types.BOOL, help="Clears caches (pipenv, pip, and pip-tools).", expose_value=False)(f) @@ -257,7 +257,7 @@ def system_option(f): state.system = value return value return option("--system", is_flag=True, default=False, help="System pip management.", - callback=callback, type=click_booltype, expose_value=False)(f) + callback=callback, type=click.types.BOOL, expose_value=False)(f) def requirementstxt_option(f): @@ -295,7 +295,7 @@ def deploy_option(f): state = ctx.ensure_object(State) state.installstate.deploy = value return value - return option("--deploy", is_flag=True, default=False, type=click_booltype, + return option("--deploy", is_flag=True, default=False, type=click.types.BOOL, help=u"Abort if the Pipfile.lock is out-of-date, or Python version is" " wrong.", callback=callback, expose_value=False)(f) diff --git a/pipenv/core.py b/pipenv/core.py index bf4145e5..7aa38b41 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1105,17 +1105,18 @@ def do_lock( # Support for --keep-outdated… if keep_outdated: + from pipenv.vendor.packaging.utils import canonicalize_name for section_name, section in ( ("default", project.packages), ("develop", project.dev_packages), ): - for package_specified in section: - norm_name = pep423_name(package_specified) + for package_specified in section.keys(): if not is_pinned(section[package_specified]): - if norm_name in cached_lockfile[section_name]: - lockfile[section_name][norm_name] = cached_lockfile[ + canonical_name = canonicalize_name(package_specified) + if canonical_name in cached_lockfile[section_name]: + lockfile[section_name][canonical_name] = cached_lockfile[ section_name - ][norm_name] + ][canonical_name].copy() # Overwrite any develop packages with default packages. for default_package in lockfile["default"]: if default_package in lockfile["develop"]: @@ -1756,9 +1757,11 @@ def do_install( if requirements or package_args or project.pipfile_exists: skip_requirements = True # Don't attempt to install develop and default packages if Pipfile is missing - if not project.pipfile_exists and not packages and dev: - click.echo("Could not find Pipfile.", err=True) - sys.exit(1) + if not project.pipfile_exists and not (packages or dev) and not code: + if not (skip_lock or deploy): + raise exceptions.PipfileNotFound(project.pipfile_location) + elif (skip_lock or deploy) and not project.lockfile_exists: + raise exceptions.LockfileNotFound(project.lockfile_location) concurrent = not sequential # Ensure that virtualenv is available. ensure_project( @@ -1778,15 +1781,7 @@ def do_install( remote = requirements and is_valid_url(requirements) # Warn and exit if --system is used without a pipfile. if (system and package_args) and not (PIPENV_VIRTUALENV): - click.echo( - "{0}: --system is intended to be used for Pipfile installation, " - "not installation of specific packages. Aborting.".format( - crayons.red("Warning", bold=True) - ), - err=True, - ) - click.echo("See also: --deploy flag.", err=True) - sys.exit(1) + raise exceptions.SystemUsageError # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True @@ -1923,6 +1918,7 @@ def do_install( pre=pre, requirements_dir=requirements_directory, pypi_mirror=pypi_mirror, + keep_outdated=keep_outdated ) # This is for if the user passed in dependencies, then we want to maek sure we @@ -1952,6 +1948,7 @@ def do_install( sys.exit(1) if index_url: pkg_requirement.index = index_url + try: c = pip_install( pkg_requirement, ignore_hashes=True, @@ -1964,6 +1961,9 @@ def do_install( extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) + except (ValueError, RuntimeError): + sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) # Warn if --editable wasn't passed. if pkg_requirement.is_vcs and not pkg_requirement.editable: sp.write_err( @@ -2215,7 +2215,8 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror click.echo(fix_utf8("Launching subshell in virtual environment…"), err=True) fork_args = (project.virtualenv_location, project.project_directory, shell_args) - + with vistir.contextmanagers.temp_environ(): + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) if fancy: shell.fork(*fork_args) return @@ -2349,6 +2350,9 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") load_dot_env() # Activate virtualenv under the current interpreter's environment + + with vistir.contextmanagers.temp_environ(): + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) inline_activate_virtual_environment() try: script = project.build_script(command, args) diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 95feb859..3fda8021 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -176,6 +176,19 @@ class PipenvOptionsError(PipenvUsageError): self.option_name = option_name +class SystemUsageError(PipenvOptionsError): + def __init__(self, option_name="system", message=None, ctx=None, **kwargs): + extra = kwargs.pop("extra", []) + extra += [ + "{0}: --system is intended to be used for Pipfile installation, " + "not installation of specific packages. Aborting.".format( + crayons.red("Warning", bold=True) + ), + ] + message = crayons.blue("See also: {0}".format(crayons.white("-deploy flag."))) + super(SystemUsageError, self).__init__(option_name, message=message, ctx=ctx, extra=extra, **kwargs) + + class PipfileException(PipenvFileError): def __init__(self, hint=None, **kwargs): from .core import project @@ -239,6 +252,16 @@ class UninstallError(PipenvException): self.extra = extra +class InstallError(PipenvException): + def __init__(self, package, **kwargs): + message = "{0} {1}".format( + crayons.red("ERROR:", bold=True), + crayons.yellow("Package installation failed...") + ) + extra = kwargs.pop("extra", []) + PipenvException.__init__(self, message=fix_utf8(message), extra=extra, **kwargs) + + class CacheError(PipenvException): def __init__(self, path, **kwargs): message = "{0} {1} {2}\n{0}".format( @@ -273,7 +296,7 @@ class ResolutionFailure(PipenvException): crayons.red("ERROR:", bold=True), crayons.yellow(message) ) if no_version_found: - messsage = "{0}\n{1}".format( + message = "{0}\n{1}".format( message, crayons.blue( "Please check your version specifier and version number. " diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index 7c69b9fc..27a5b3fc 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -7,7 +7,9 @@ import sys PYENV_INSTALLED = bool(os.environ.get("PYENV_SHELL")) or bool( os.environ.get("PYENV_ROOT") ) -PYENV_ROOT = os.path.expandvars(os.environ.get("PYENV_ROOT", "~/.pyenv")) +PYENV_ROOT = os.path.expanduser( + os.path.expandvars(os.environ.get("PYENV_ROOT", "~/.pyenv")) +) IS_64BIT_OS = None SYSTEM_ARCH = platform.architecture()[0] diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 5874af64..4a8dfc65 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -71,7 +71,7 @@ class PyenvFinder(BaseFinder, BasePath): @versions.default def get_versions(self): versions = defaultdict() - bin_ = sysconfig._INSTALL_SCHEMES[sysconfig._get_default_scheme()]["scripts"] + bin_ = sysconfig._INSTALL_SCHEMES['posix_prefix']["scripts"] for p in self.get_version_order(): bin_dir = Path(bin_.format(base=p.as_posix())) version_path = None diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index c8a995fa..f3554d56 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -31,7 +31,7 @@ from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfi from .spin import VistirSpinner, create_spinner -__version__ = "0.2.3" +__version__ = "0.2.4" __all__ = [ diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index 20587d9d..f8c4e009 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -153,11 +153,15 @@ class VistirSpinner(base_obj): def ok(self, text="OK"): """Set Ok (success) finalizer to a spinner.""" + self._text = None + _text = text if text else "OK" self._freeze(_text) def fail(self, text="FAIL"): """Set fail finalizer to a spinner.""" + self._text = None + _text = text if text else "FAIL" self._freeze(_text) From e398d348b455fa1d24ce273f5ee431197bc88967 Mon Sep 17 00:00:00 2001 From: David Staheli Date: Mon, 5 Nov 2018 11:22:34 -0500 Subject: [PATCH 073/218] Added news file --- news/3164.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/3164.bugfix.rst diff --git a/news/3164.bugfix.rst b/news/3164.bugfix.rst new file mode 100644 index 00000000..7b3fb1ac --- /dev/null +++ b/news/3164.bugfix.rst @@ -0,0 +1 @@ +Azure Pipelines YAML files are updated to use the latest syntax and product name. From 6b2f3b66a65b0966da48ce612995fb0ee6a36da9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 5 Nov 2018 11:30:41 -0500 Subject: [PATCH 074/218] Syntax error fix Signed-off-by: Dan Ryan --- pipenv/core.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index d03ba042..26ac4d0e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1949,18 +1949,18 @@ def do_install( if index_url: pkg_requirement.index = index_url try: - c = pip_install( - pkg_requirement, - ignore_hashes=True, - allow_global=system, - selective_upgrade=selective_upgrade, - no_deps=False, - pre=pre, - requirements_dir=requirements_directory, - index=index_url, - extra_indexes=extra_index_url, - pypi_mirror=pypi_mirror, - ) + c = pip_install( + pkg_requirement, + ignore_hashes=True, + allow_global=system, + selective_upgrade=selective_upgrade, + no_deps=False, + pre=pre, + requirements_dir=requirements_directory, + index=index_url, + extra_indexes=extra_index_url, + pypi_mirror=pypi_mirror, + ) except (ValueError, RuntimeError): sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) From 363ffc64085bd13749970052201738b5e807eebc Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 5 Nov 2018 11:36:33 -0500 Subject: [PATCH 075/218] Remove unfinished validation Signed-off-by: Dan Ryan --- pipenv/cli/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index b701ef90..277c014a 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -103,7 +103,7 @@ def extra_index_option(f): def editable_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) - state.installstate.editables.extend(validate_requirements(ctx, param, value)) + state.installstate.editables.extend(value) return value return option('-e', '--editable', expose_value=False, multiple=True, help='An editable python package URL or path, often to a VCS repo.', @@ -182,7 +182,7 @@ def pre_option(f): def package_arg(f): def callback(ctx, param, value): state = ctx.ensure_object(State) - state.installstate.packages.extend(validate_requirements(ctx, param, value)) + state.installstate.packages.extend(value) return value return argument('packages', nargs=-1, callback=callback, expose_value=False,)(f) From d773429d0721f1aa56bc8f3ae85147d5f681419d Mon Sep 17 00:00:00 2001 From: David Staheli Date: Mon, 5 Nov 2018 11:36:53 -0500 Subject: [PATCH 076/218] Update MANIFEST.in to prune .azure-pipelines --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index a8d08c6c..3c8eb1d4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -28,6 +28,7 @@ recursive-exclude pipenv *.pyi recursive-exclude pipenv *.typed prune peeps +prune .azure-pipelines prune .buildkite prune .github prune .vsts-ci From db7164d988cb4f82198e8f03d7cac9580258d271 Mon Sep 17 00:00:00 2001 From: jxltom Date: Tue, 6 Nov 2018 12:40:44 +0800 Subject: [PATCH 077/218] Skip purging if there is no packages needs to be removed --- pipenv/core.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pipenv/core.py b/pipenv/core.py index 2091bd08..1922b28f 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1158,6 +1158,13 @@ def do_purge(bare=False, downloads=False, allow_global=False): click.echo( fix_utf8("Found {0} installed package(s), purging…".format(len(to_remove))) ) + + # Skip purging if there is no packages which needs to be removed + if not to_remove: + if not bare: + click.echo(crayons.green("Environment now purged and fresh!")) + return installed + command = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=allow_global)), " ".join(to_remove), From e53391fd49aba7663e24d747b53c8f808225e668 Mon Sep 17 00:00:00 2001 From: jxltom Date: Tue, 6 Nov 2018 12:46:52 +0800 Subject: [PATCH 078/218] Add feature news for issue 3170 --- news/3170.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/3170.feature diff --git a/news/3170.feature b/news/3170.feature new file mode 100644 index 00000000..b8bc5218 --- /dev/null +++ b/news/3170.feature @@ -0,0 +1 @@ +Do not show error but success for running pipenv uninstall --all in a fresh virtuanlenv From ecec1ac0f9352af002e5790ae4fc4fc82a9a2a59 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 6 Nov 2018 14:14:55 +0900 Subject: [PATCH 079/218] Just format --- pipenv/cli/command.py | 18 ++++++---- pipenv/core.py | 69 +++++++++++++---------------------- pipenv/utils.py | 84 +++++++++++++++++++++++++++++-------------- 3 files changed, 94 insertions(+), 77 deletions(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 74f66d7e..74c57720 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -98,12 +98,12 @@ def cli( warn_in_virtualenv, do_where, project, - spinner, cleanup_virtualenv, ensure_project, format_help, do_clear, ) + from ..utils import create_spinner if man: if system_which("man"): @@ -179,7 +179,7 @@ def cli( ) ) ) - with spinner(): + with create_spinner(text="Running..."): # Remove the virtualenv. cleanup_virtualenv(bare=True) return 0 @@ -312,8 +312,12 @@ def lock( # Ensure that virtualenv is available. ensure_project(three=state.three, python=state.python, pypi_mirror=state.pypi_mirror) if state.installstate.requirementstxt: - do_init(dev=state.installstate.dev, requirements=state.installstate.requirementstxt, - pypi_mirror=state.pypi_mirror, pre=state.installstate.pre) + do_init( + dev=state.installstate.dev, + requirements=state.installstate.requirementstxt, + pypi_mirror=state.pypi_mirror, + pre=state.installstate.pre, + ) do_lock( ctx=ctx, clear=state.clear, @@ -549,8 +553,10 @@ def run_open(state, module, *args, **kwargs): from ..core import which, ensure_project # Ensure that virtualenv is available. - ensure_project(three=state.three, python=state.python, validate=False, - pypi_mirror=state.pypi_mirror) + ensure_project( + three=state.three, python=state.python, + validate=False, pypi_mirror=state.pypi_mirror, + ) c = delegator.run( '{0} -c "import {1}; print({1}.__file__);"'.format(which("python"), module) ) diff --git a/pipenv/core.py b/pipenv/core.py index 2091bd08..d0136671 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1,6 +1,5 @@ # -*- coding=utf-8 -*- -import contextlib import logging import os import sys @@ -18,7 +17,6 @@ import warnings import six import urllib3.util as urllib3_util -from functools import partial from .cmdparse import Script from .project import Project, SourceNotFound @@ -42,7 +40,8 @@ from .utils import ( clean_resolved_dep, parse_indexes, escape_cmd, - fix_venv_site + fix_venv_site, + create_spinner, ) from . import environments, pep508checker, progress from .environments import ( @@ -101,26 +100,6 @@ if PIPENV_COLORBLIND: crayons.disable() -@contextlib.contextmanager -def _spinner(text=None, nospin=None, spinner_name=None): - if not text: - text = "Running..." - if not spinner_name: - spinner_name = environments.PIPENV_SPINNER - if nospin is None: - nospin = environments.PIPENV_NOSPIN - with vistir.spin.create_spinner( - spinner_name=spinner_name, - start_text=text, - nospin=nospin - ) as sp: - yield sp - - -spinner = partial(_spinner, text="Running...", nospin=environments.PIPENV_NOSPIN, - spinner_name=environments.PIPENV_SPINNER) - - def which(command, location=None, allow_global=False): if not allow_global and location is None: if project.virtualenv_exists: @@ -300,7 +279,8 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): if project.pipfile_is_empty: # Show an error message and exit if system is passed and no pipfile exists if system and not PIPENV_VIRTUALENV: - raise exceptions.PipenvOptionsError("--system", + raise exceptions.PipenvOptionsError( + "--system", "--system is intended to be used for pre-existing Pipfile " "installation, not installation of specific packages. Aborting." ) @@ -314,9 +294,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): ) # Create a Pipfile… project.create_pipfile(python=python) - with spinner(text=vistir.compat.fs_str("Importing requirements..."), - spinner_name=environments.PIPENV_SPINNER, - nospin=environments.PIPENV_NOSPIN) as sp: + with create_spinner("Importing requirements...") as sp: # Import requirements.txt. try: import_requirements() @@ -466,9 +444,7 @@ def ensure_python(three=None, python=None): crayons.normal(fix_utf8("…"), bold=True), ) ) - with spinner(text=vistir.compat.fs_str("Installing python..."), - spinner_name=environments.PIPENV_SPINNER, - nospin=environments.PIPENV_NOSPIN) as sp: + with create_spinner("Installing python...") as sp: try: c = pyenv.install(version) except PyenvError as e: @@ -762,8 +738,10 @@ def do_install_dependencies( procs = queue.Queue(maxsize=PIPENV_MAX_SUBPROCESS) trusted_hosts = [] - deps_list_bar = progress.bar(deps_list, width=32, - label=INSTALL_LABEL if os.name != "nt" else "") + deps_list_bar = progress.bar( + deps_list, width=32, + label=INSTALL_LABEL if os.name != "nt" else "", + ) indexes = [] for dep in deps_list_bar: index = None @@ -921,9 +899,11 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): # Actually create the virtualenv. nospin = environments.PIPENV_NOSPIN - c = vistir.misc.run(cmd, verbose=False, return_object=True, - spinner_name=environments.PIPENV_SPINNER, combine_stderr=False, - block=False, nospin=nospin, env=pip_config) + c = vistir.misc.run( + cmd, verbose=False, return_object=True, + spinner_name=environments.PIPENV_SPINNER, combine_stderr=False, + block=False, nospin=nospin, env=pip_config, + ) click.echo(crayons.blue("{0}".format(c.out)), err=True) if c.returncode != 0: raise exceptions.VirtualenvCreationException( @@ -1253,7 +1233,8 @@ def do_init( # Unless we're in a virtualenv not managed by pipenv, abort if we're # using the system's python. if (system or allow_global) and not (PIPENV_VIRTUALENV): - raise exceptions.PipenvOptionsError("--system", + raise exceptions.PipenvOptionsError( + "--system", "--system is intended to be used for Pipfile installation, " "not installation of specific packages. Aborting.\n" "See also: --deploy flag." @@ -1940,9 +1921,8 @@ def do_install( ) ) # pip install: - with vistir.contextmanagers.temp_environ(), spinner(text="Installing...", - spinner_name=environments.PIPENV_SPINNER, - nospin=environments.PIPENV_NOSPIN) as sp: + with vistir.contextmanagers.temp_environ(), \ + create_spinner("Installing...") as sp: os.environ["PIP_USER"] = vistir.compat.fs_str("0") try: pkg_requirement = Requirement.from_line(pkg_line) @@ -2040,7 +2020,6 @@ def do_uninstall( ): from .environments import PIPENV_USE_SYSTEM from .vendor.requirementslib.models.requirements import Requirement - from .vendor.requirementslib.models.lockfile import Lockfile from .vendor.packaging.utils import canonicalize_name # Automatically use an activated virtualenv. @@ -2052,8 +2031,10 @@ def do_uninstall( ensure_project(three=three, python=python, pypi_mirror=pypi_mirror) # Un-install all dependencies, if --all was provided. if not any([packages, editable_packages, all_dev, all]): - raise exceptions.MissingParameter(crayons.red("No package provided!"), ctx=ctx, - param_type="parameter") + raise exceptions.MissingParameter( + crayons.red("No package provided!"), + ctx=ctx, param_type="parameter", + ) editable_pkgs = [ Requirement.from_line("-e {0}".format(p)).name for p in editable_packages if p ] @@ -2151,8 +2132,8 @@ def do_uninstall( # Uninstall the package. if package_name in packages_to_remove: cmd = "{0} uninstall {1} -y".format( - escape_grouped_arguments(which_pip(allow_global=system)), package_name - ) + escape_grouped_arguments(which_pip(allow_global=system)), package_name, + ) if environments.is_verbose(): click.echo("$ {0}".format(cmd)) c = delegator.run(cmd) diff --git a/pipenv/utils.py b/pipenv/utils.py index c0d5a356..c9feeafd 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1,16 +1,17 @@ # -*- coding: utf-8 -*- +import contextlib import errno import logging import os import re import shutil +import stat import sys +import warnings import crayons import parse import six -import stat -import warnings from click import echo as click_echo from first import first @@ -363,17 +364,31 @@ class Resolver(object): def populate_file_hashes(self): from pipenv.vendor.vistir.compat import Path, to_native_string from pipenv.vendor.vistir.path import url_to_path + + def _should_include(ireq): + # We can only hash artifacts. + try: + if not ireq.link.is_artifact: + return False + except AttributeError: + return False + + # But we don't want normal pypi artifcats since the normal resolver + # handles those + if is_pypi_url(ireq.link.url): + return False + + # We also don't want to try to hash directories as this will fail + # as these are editable deps and are not hashable. + if (ireq.link.scheme == "file" and + Path(to_native_string(url_to_path(ireq.link.url))).is_dir()): + return False + return True + self.hashes.update({ ireq: self.resolver._hash_cache.get_hash(ireq.link) - for ireq in self.parsed_constraints if (getattr(ireq, "link", None) - # We can only hash artifacts, but we don't want normal pypi artifcats since the - # normal resolver handles those - and ireq.link.is_artifact and not (is_pypi_url(ireq.link.url) or ( - # We also don't want to try to hash directories as this will fail - # as these are editable deps and are not hashable - ireq.link.scheme == "file" and - Path(to_native_string(url_to_path(ireq.link.url))).is_dir() - ))) + for ireq in self.parsed_constraints + if _should_include(ireq) }) @property @@ -428,8 +443,9 @@ def actually_resolve_deps( if not req_dir: req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") - constraints = get_resolver_metadata(deps, index_lookup, markers_lookup, project, - sources) + constraints = get_resolver_metadata( + deps, index_lookup, markers_lookup, project, sources, + ) resolver = Resolver(constraints, req_dir, project, sources, clear=clear, pre=pre) resolved_tree = resolver.resolve() hashes = resolver.resolve_hashes() @@ -437,6 +453,21 @@ def actually_resolve_deps( return (resolved_tree, hashes, markers_lookup, resolver) +@contextlib.contextmanager +def create_spinner(text, nospin=None, spinner_name=None): + import vistir.spin + if not spinner_name: + spinner_name = environments.PIPENV_SPINNER + if nospin is None: + nospin = environments.PIPENV_NOSPIN + with vistir.spin.create_spinner( + spinner_name=spinner_name, + start_text=vistir.compat.fs_str(text), + nospin=nospin + ) as sp: + yield sp + + def venv_resolve_deps( deps, which, @@ -450,7 +481,6 @@ def venv_resolve_deps( from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError from .vendor.vistir.path import create_tracked_tempdir from .cmdparse import Script - from .core import spinner from .vendor.pexpect.exceptions import EOF, TIMEOUT from .vendor import delegator from . import resolver @@ -481,8 +511,7 @@ def venv_resolve_deps( os.environ["PIP_NO_INPUT"] = fs_str("1") out = to_native_string("") EOF.__module__ = "pexpect.exceptions" - with spinner(text=fs_str("Locking..."), spinner_name=environments.PIPENV_SPINNER, - nospin=environments.PIPENV_NOSPIN) as sp: + with create_spinner(text=fs_str("Locking...")) as sp: c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) _out = decode_output("") result = None @@ -1221,8 +1250,6 @@ def get_vcs_deps( dev=False, pypi_mirror=None, ): - from .vendor.vistir.compat import Path - from .vendor.vistir.path import create_tracked_tempdir from .vendor.requirementslib.models.requirements import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" @@ -1280,8 +1307,10 @@ def translate_markers(pipfile_entry): marker_set.add(str(Marker("{0}{1}".format(m, entry)))) new_pipfile.pop(m) if marker_set: - new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s) if " and " in s else s - for s in sorted(dedup(marker_set))]))).replace('"', "'") + new_pipfile["markers"] = str(Marker(" or ".join( + "{0}".format(s) if " and " in s else s + for s in sorted(dedup(marker_set)) + ))).replace('"', "'") return new_pipfile @@ -1289,9 +1318,6 @@ def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None): name = pep423_name(dep["name"]) # We use this to determine if there are any markers on top level packages # So we can make sure those win out during resolution if the packages reoccur - dep_keys = ( - [k for k in getattr(pipfile_entry, "keys", list)()] if is_top_level else [] - ) lockfile = {"version": "=={0}".format(dep["version"])} for key in ["hashes", "index", "extras"]: if key in dep: @@ -1406,10 +1432,14 @@ def looks_like_dir(path): def parse_indexes(line): from argparse import ArgumentParser parser = ArgumentParser("indexes") - parser.add_argument("--index", "-i", "--index-url", metavar="index_url", - action="store", nargs="?",) - parser.add_argument("--extra-index-url", "--extra-index", metavar="extra_indexes", - action="append") + parser.add_argument( + "--index", "-i", "--index-url", + metavar="index_url", action="store", nargs="?", + ) + parser.add_argument( + "--extra-index-url", "--extra-index", + metavar="extra_indexes",action="append", + ) parser.add_argument("--trusted-host", metavar="trusted_hosts", action="append") args, remainder = parser.parse_known_args(line.split()) index = [] if not args.index else [args.index,] From e61faeba40cc6e46f5ea26227215e9fc79f41e21 Mon Sep 17 00:00:00 2001 From: jxltom Date: Tue, 6 Nov 2018 14:22:55 +0800 Subject: [PATCH 080/218] Do not touch pipfile for --all option in pipenv uninstall --- pipenv/core.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 2091bd08..4cfae2e3 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2118,18 +2118,6 @@ def do_uninstall( crayons.normal(fix_utf8("Un-installing all packages from virtualenv…"), bold=True) ) do_purge(allow_global=system) - removed = package_names - bad_pkgs - if pipfile_remove: - project.remove_packages_from_pipfile(removed) - if lock: - do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) - else: - lockfile = project.get_or_create_lockfile() - for key in lockfile.default.keys(): - del lockfile.default[key] - for key in lockfile.develop.keys(): - del lockfile.develop[key] - lockfile.write() return if all_dev: package_names = develop From f9ee0c80ca29f805db7b24e4465b2364f7b265f5 Mon Sep 17 00:00:00 2001 From: jxltom Date: Tue, 6 Nov 2018 14:23:16 +0800 Subject: [PATCH 081/218] Do not add skip-lock for lock command --- pipenv/cli/options.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index dcdc1d9f..99fc5344 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -353,7 +353,6 @@ def uninstall_options(f): def lock_options(f): f = install_base_options(f) f = requirements_flag(f) - f = skip_lock_option(f) f = pre_option(f) return f From 41023c33ae780b8b6621e0e081615b21e82e783f Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 6 Nov 2018 15:46:14 +0900 Subject: [PATCH 082/218] Upgrade some requirements to address security warning --- docs/requirements.txt | 2 +- tests/pytest-pypi/Pipfile | 23 -- tests/pytest-pypi/Pipfile.lock | 480 --------------------------------- 3 files changed, 1 insertion(+), 504 deletions(-) delete mode 100644 tests/pytest-pypi/Pipfile delete mode 100644 tests/pytest-pypi/Pipfile.lock diff --git a/docs/requirements.txt b/docs/requirements.txt index 71c58644..d35a9f32 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -15,7 +15,7 @@ pip-tools==1.9.0 Pygments==2.2.0 pythonz-bd==1.11.4 pytz==2017.2 -requests==2.12.0 +requests==2.20.0 resumable-urlretrieve==0.1.5 semver==2.7.8 six==1.10.0 diff --git a/tests/pytest-pypi/Pipfile b/tests/pytest-pypi/Pipfile deleted file mode 100644 index 6abe710c..00000000 --- a/tests/pytest-pypi/Pipfile +++ /dev/null @@ -1,23 +0,0 @@ -[[source]] - -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - - -[packages] - - - -[dev-packages] - -pytest = "*" -"e1839a8" = {path = ".", editable = true} -httpbin = "*" -requests = "*" -twine = "*" - - -[requires] - -python_version = "3.6" diff --git a/tests/pytest-pypi/Pipfile.lock b/tests/pytest-pypi/Pipfile.lock deleted file mode 100644 index 25978b9d..00000000 --- a/tests/pytest-pypi/Pipfile.lock +++ /dev/null @@ -1,480 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "b871e70f9f0b7bd3852672c371c754d8db50e5015c8dbdad1d217a3e58a26c33" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.6" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": {}, - "develop": { - "argparse": { - "hashes": [ - "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", - "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" - ], - "markers": "python_version == '2.6'", - "version": "==1.4.0" - }, - "attrs": { - "hashes": [ - "sha256:1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9", - "sha256:a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450" - ], - "version": "==17.4.0" - }, - "blinker": { - "hashes": [ - "sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6" - ], - "version": "==1.4" - }, - "brotlipy": { - "hashes": [ - "sha256:07194f4768eb62a4f4ea76b6d0df6ade185e24ebd85877c351daa0a069f1111a", - "sha256:091b299bf36dd6ef7a06570dbc98c0f80a504a56c5b797f31934d2ad01ae7d17", - "sha256:09ec3e125d16749b31c74f021aba809541b3564e5359f8c265cbae442810b41a", - "sha256:0be698678a114addcf87a4b9496c552c68a2c99bf93cf8e08f5738b392e82057", - "sha256:0fa6088a9a87645d43d7e21e32b4a6bf8f7c3939015a50158c10972aa7f425b7", - "sha256:1ea4e578241504b58f2456a6c69952c88866c794648bdc74baee74839da61d44", - "sha256:2699945a0a992c04fc7dc7fa2f1d0575a2c8b4b769f2874a08e8eae46bef36ae", - "sha256:2a80319ae13ea8dd60ecdc4f5ccf6da3ae64787765923256b62c598c5bba4121", - "sha256:2e5c64522364a9ebcdf47c5744a5ddeb3f934742d31e61ebfbbc095460b47162", - "sha256:36def0b859beaf21910157b4c33eb3b06d8ce459c942102f16988cca6ea164df", - "sha256:3a3e56ced8b15fbbd363380344f70f3b438e0fd1fcf27b7526b6172ea950e867", - "sha256:3c1d5e2cf945a46975bdb11a19257fa057b67591eb232f393d260e7246d9e571", - "sha256:50ca336374131cfad20612f26cc43c637ac0bfd2be3361495e99270883b52962", - "sha256:5de6f7d010b7558f72f4b061a07395c5c3fd57f0285c5af7f126a677b976a868", - "sha256:637847560d671657f993313ecc6c6c6666a936b7a925779fd044065c7bc035b9", - "sha256:653faef61241bf8bf99d73ca7ec4baa63401ba7b2a2aa88958394869379d67c7", - "sha256:786afc8c9bd67de8d31f46e408a3386331e126829114e4db034f91eacb05396d", - "sha256:79aaf217072840f3e9a3b641cccc51f7fc23037496bd71e26211856b93f4b4cb", - "sha256:7e31f7adcc5851ca06134705fcf3478210da45d35ad75ec181e1ce9ce345bb38", - "sha256:8b39abc3256c978f575df5cd7893153277216474f303e26f0e43ba3d3969ef96", - "sha256:9448227b0df082e574c45c983fa5cd4bda7bfb11ea6b59def0940c1647be0c3c", - "sha256:96bc59ff9b5b5552843dc67999486a220e07a0522dddd3935da05dc194fa485c", - "sha256:a07647886e24e2fb2d68ca8bf3ada398eb56fd8eac46c733d4d95c64d17f743b", - "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2", - "sha256:b4c98b0d2c9c7020a524ca5bbff42027db1004c6571f8bc7b747f2b843128e7a", - "sha256:c6cc0036b1304dd0073eec416cb2f6b9e37ac8296afd9e481cac3b1f07f9db25", - "sha256:d2c1c724c4ac375feb2110f1af98ecdc0e5a8ea79d068efb5891f621a5b235cb", - "sha256:dc6c5ee0df9732a44d08edab32f8a616b769cc5a4155a12d2d010d248eb3fb07", - "sha256:fd1d1c64214af5d90014d82cee5d8141b13d44c92ada7a0c0ec0679c6f15a471" - ], - "version": "==0.7.0" - }, - "certifi": { - "hashes": [ - "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296", - "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d" - ], - "version": "==2018.1.18" - }, - "cffi": { - "hashes": [ - "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", - "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", - "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", - "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", - "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", - "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", - "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", - "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", - "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", - "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", - "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", - "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", - "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", - "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", - "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", - "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", - "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", - "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", - "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", - "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", - "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", - "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", - "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", - "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", - "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", - "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", - "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" - ], - "version": "==1.11.5" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", - "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" - ], - "version": "==6.7" - }, - "colorama": { - "hashes": [ - "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", - "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" - ], - "markers": "sys_platform == 'win32'", - "version": "==0.3.9" - }, - "contextlib2": { - "hashes": [ - "sha256:509f9419ee91cdd00ba34443217d5ca51f5a364a404e1dce9e8979cea969ca48", - "sha256:f5260a6e679d2ff42ec91ec5252f4eeffdcf21053db9113bd0a8e4d953769c00" - ], - "markers": "python_version < '3.2'", - "version": "==0.5.5" - }, - "crayons": { - "hashes": [ - "sha256:5e17691605e564d63482067eb6327d01a584bbaf870beffd4456a3391bd8809d", - "sha256:6f51241d0c4faec1c04c1c0ac6a68f1d66a4655476ce1570b3f37e5166a599cc" - ], - "version": "==0.1.2" - }, - "dateparser": { - "hashes": [ - "sha256:940828183c937bcec530753211b70f673c0a9aab831e43273489b310538dff86", - "sha256:b452ef8b36cd78ae86a50721794bc674aa3994e19b570f7ba92810f4e0a2ae03" - ], - "version": "==0.7.0" - }, - "decorator": { - "hashes": [ - "sha256:7d46dd9f3ea1cf5f06ee0e4e1277ae618cf48dfb10ada7c8427cd46c42702a0e", - "sha256:94d1d8905f5010d74bbbd86c30471255661a14187c45f8d7f3e5aa8540fdb2e5" - ], - "version": "==4.2.1" - }, - "e1839a8": { - "editable": true, - "path": "." - }, - "flask": { - "hashes": [ - "sha256:0749df235e3ff61ac108f69ac178c9770caeaccad2509cb762ce1f65570a8856", - "sha256:49f44461237b69ecd901cc7ce66feea0319b9158743dd27a2899962ab214dac1" - ], - "version": "==0.12.2" - }, - "flask-cache": { - "hashes": [ - "sha256:33187b3ddceeee233fe3db68ffcc118b5498e8ad28edde711bcbdcbf4924ce35", - "sha256:90126ca9bc063854ef8ee276e95d38b2b4ec8e45fd77d5751d37971ee27c7ef4", - "sha256:ae9d1ac4549517dfbc1f178ccc5429f61f836be3cc109a0b2481c98b3711c329" - ], - "version": "==0.13.1" - }, - "flask-common": { - "hashes": [ - "sha256:44fbb57a12bc7478d56c223eb5de7b2fb98ce42a70314c74ffecf5dbe75ed1b8" - ], - "version": "==0.2.0" - }, - "flask-limiter": { - "hashes": [ - "sha256:473aa5bc97310406aa8c12ab3dc080697bcfa8cd21a6d0aba30916911bbc673c", - "sha256:8cce98dcf25bf2ddbb824c2b503b4fc8e1a139154240fd2c60d9306bad8a0db8" - ], - "version": "==1.0.1" - }, - "funcsigs": { - "hashes": [ - "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", - "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" - ], - "markers": "python_version < '3.0'", - "version": "==1.0.2" - }, - "greenlet": { - "hashes": [ - "sha256:09ef2636ea35782364c830f07127d6c7a70542b178268714a9a9ba16318e7e8b", - "sha256:0fef83d43bf87a5196c91e73cb9772f945a4caaff91242766c5916d1dd1381e4", - "sha256:1b7df09c6598f5cfb40f843ade14ed1eb40596e75cd79b6fa2efc750ba01bb01", - "sha256:1fff21a2da5f9e03ddc5bd99131a6b8edf3d7f9d6bc29ba21784323d17806ed7", - "sha256:42118bf608e0288e35304b449a2d87e2ba77d1e373e8aa221ccdea073de026fa", - "sha256:50643fd6d54fd919f9a0a577c5f7b71f5d21f0959ab48767bd4bb73ae0839500", - "sha256:58798b5d30054bb4f6cf0f712f08e6092df23a718b69000786634a265e8911a9", - "sha256:5b49b3049697aeae17ef7bf21267e69972d9e04917658b4e788986ea5cc518e8", - "sha256:75c413551a436b462d5929255b6dc9c0c3c2b25cbeaee5271a56c7fda8ca49c0", - "sha256:769b740aeebd584cd59232be84fdcaf6270b8adc356596cdea5b2152c82caaac", - "sha256:ad2383d39f13534f3ca5c48fe1fc0975676846dc39c2cece78c0f1f9891418e0", - "sha256:b417bb7ff680d43e7bd7a13e2e08956fa6acb11fd432f74c97b7664f8bdb6ec1", - "sha256:b6ef0cabaf5a6ecb5ac122e689d25ba12433a90c7b067b12e5f28bdb7fb78254", - "sha256:c2de19c88bdb0366c976cc125dca1002ec1b346989d59524178adfd395e62421", - "sha256:c7b04a6dc74087b1598de8d713198de4718fa30ec6cbb84959b26426c198e041", - "sha256:f8f2a0ae8de0b49c7b5b2daca4f150fdd9c1173e854df2cce3b04123244f9f45", - "sha256:fcfadaf4bf68a27e5dc2f42cbb2f4b4ceea9f05d1d0b8f7787e640bed2801634" - ], - "version": "==0.4.13" - }, - "gunicorn": { - "hashes": [ - "sha256:75af03c99389535f218cc596c7de74df4763803f7b63eb09d77e92b3956b36c6", - "sha256:eee1169f0ca667be05db3351a0960765620dad53f53434262ff8901b68a1b622" - ], - "version": "==19.7.1" - }, - "httpbin": { - "hashes": [ - "sha256:0afa0486a76305cac441b5cc80d5d4ccd82b20875da7c5119ecfe616cefef45f", - "sha256:7c3a4d69f9d495e6df1d51a50ba7b1e4f33a1cb61e0db95ee7f8953e27a2243f" - ], - "version": "==0.6.2" - }, - "humanize": { - "hashes": [ - "sha256:a43f57115831ac7c70de098e6ac46ac13be00d69abbf60bdcac251344785bb19" - ], - "version": "==0.5.1" - }, - "idna": { - "hashes": [ - "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f", - "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4" - ], - "version": "==2.6" - }, - "itsdangerous": { - "hashes": [ - "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" - ], - "version": "==0.24" - }, - "jinja2": { - "hashes": [ - "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", - "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" - ], - "version": "==2.10" - }, - "limits": { - "hashes": [ - "sha256:9df578f4161017d79f5188609f1d65f6b639f8aad2914c3960c9252e56a0ff95", - "sha256:a017b8d9e9da6761f4574642149c337f8f540d4edfe573fb91ad2c4001a2bc76" - ], - "version": "==1.3" - }, - "markupsafe": { - "hashes": [ - "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665" - ], - "version": "==1.0" - }, - "maya": { - "hashes": [ - "sha256:ad1969bae78afb148c45a2f63591a7575ec05b4a0ab7ec04987ab7d73649f9d6", - "sha256:d8a7ed8513b2990036fe456c9f595b54d19ec49cb4461cd95a2ef6c487fb55eb" - ], - "version": "==0.3.4" - }, - "meinheld": { - "hashes": [ - "sha256:293eff4983b7fcbd9134b47706b22189883fe354993bd10163c65869d141e565", - "sha256:40d9dbce0165b2d9142f364d26fd6d59d3682f89d0dfe2117717a8ddad1f4133" - ], - "version": "==0.6.1" - }, - "pendulum": { - "hashes": [ - "sha256:0c14388546db6605a860b8b7112cb69d0b11c9ce5e072210504544e0d4575799", - "sha256:39a255776528afe11ea0d57814f9bf3729c1e0b99063af2e5c6cfd750c3e1f7f", - "sha256:3c85e8cbc91f45e1cc916cc9180b34153cd6aaaaacfb51a48b3156318314fa82", - "sha256:8199206c479b13947dcac63c025575d035331bb3819d1783dc1d568a11962906", - "sha256:8798aeca58b3dd7ffdc5a4993c9eaafedc4048165429e8f499ddd62c73bf3964", - "sha256:881efe37328de0785c0731d462e1485a45712f2cd5cb55907d6c15458460ebeb", - "sha256:bcca072f82e84b419efec1320cd3ee5c230d263f3a601b146651ed4db77d89f0", - "sha256:ff0c5fa3af4a471a218408c448b804ac6bccb105127727474f4e83c0e4072e97" - ], - "version": "==1.4.2" - }, - "pkginfo": { - "hashes": [ - "sha256:31a49103180ae1518b65d3f4ce09c784e2bc54e338197668b4fb7dc539521024", - "sha256:bb1a6aeabfc898f5df124e7e00303a5b3ec9a489535f346bfbddb081af93f89e" - ], - "version": "==1.4.1" - }, - "pluggy": { - "hashes": [ - "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff" - ], - "version": "==0.6.0" - }, - "py": { - "hashes": [ - "sha256:8cca5c229d225f8c1e3085be4fcf306090b00850fefad892f9d96c7b6e2f310f", - "sha256:ca18943e28235417756316bfada6cd96b23ce60dd532642690dcfdaba988a76d" - ], - "version": "==1.5.2" - }, - "pycparser": { - "hashes": [ - "sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226" - ], - "version": "==2.18" - }, - "pytest": { - "hashes": [ - "sha256:8970e25181e15ab14ae895599a0a0e0ade7d1f1c4c8ca1072ce16f25526a184d", - "sha256:9ddcb879c8cc859d2540204b5399011f842e5e8823674bf429f70ada281b3cc6" - ], - "version": "==3.4.1" - }, - "python-dateutil": { - "hashes": [ - "sha256:891c38b2a02f5bb1be3e4793866c8df49c7d19baabf9c1bad62547e0b4866aca", - "sha256:95511bae634d69bc7329ba55e646499a842bc4ec342ad54a8cdb65645a0aad3c" - ], - "version": "==2.6.1" - }, - "pytz": { - "hashes": [ - "sha256:07edfc3d4d2705a20a6e99d97f0c4b61c800b8232dc1c04d87e8554f130148dd", - "sha256:3a47ff71597f821cd84a162e71593004286e5be07a340fd462f0d33a760782b5", - "sha256:410bcd1d6409026fbaa65d9ed33bf6dd8b1e94a499e32168acfc7b332e4095c0", - "sha256:5bd55c744e6feaa4d599a6cbd8228b4f8f9ba96de2c38d56f08e534b3c9edf0d", - "sha256:61242a9abc626379574a166dc0e96a66cd7c3b27fc10868003fa210be4bff1c9", - "sha256:887ab5e5b32e4d0c86efddd3d055c1f363cbaa583beb8da5e22d2fa2f64d51ef", - "sha256:ba18e6a243b3625513d85239b3e49055a2f0318466e0b8a92b8fb8ca7ccdf55f", - "sha256:ed6509d9af298b7995d69a440e2822288f2eca1681b8cce37673dbb10091e5fe", - "sha256:f93ddcdd6342f94cea379c73cddb5724e0d6d0a1c91c9bdef364dc0368ba4fda" - ], - "version": "==2018.3" - }, - "pytzdata": { - "hashes": [ - "sha256:4e2cceb54335cd6c28caea46b15cd592e2aec5e8b05b0241cbccfb1b23c02ae7", - "sha256:7cd949123e2c2060fd12793de3a4a449e36b5dea5e169b810a3ac3f0b9877cfa" - ], - "version": "==2018.3" - }, - "raven": { - "hashes": [ - "sha256:738a52019d01955d5b44b49d67c9f2f4cedb1b4f70d4fb0b493931174d00e044", - "sha256:92bf4c4819472ed20f1b9905eeeafe1bc6fe5f273d7c14506fdb8fb3a6ab2074" - ], - "version": "==6.6.0" - }, - "regex": { - "hashes": [ - "sha256:1b428a296531ea1642a7da48562746309c5c06471a97bd0c02dd6a82e9cecee8", - "sha256:27d72bb42dffb32516c28d218bb054ce128afd3e18464f30837166346758af67", - "sha256:32cf4743debee9ea12d3626ee21eae83052763740e04086304e7a74778bf58c9", - "sha256:32f6408dbca35040bc65f9f4ae1444d5546411fde989cb71443a182dd643305e", - "sha256:333687d9a44738c486735955993f83bd22061a416c48f5a5f9e765e90cf1b0c9", - "sha256:35eeccf17af3b017a54d754e160af597036435c58eceae60f1dd1364ae1250c7", - "sha256:361a1fd703a35580a4714ec28d85e29780081a4c399a99bbfb2aee695d72aedb", - "sha256:494bed6396a20d3aa6376bdf2d3fbb1005b8f4339558d8ac7b53256755f80303", - "sha256:5b9c0ddd5b4afa08c9074170a2ea9b34ea296e32aeea522faaaaeeeb2fe0af2e", - "sha256:a50532f61b23d4ab9d216a6214f359dd05c911c1a1ad20986b6738a782926c1a", - "sha256:a9243d7b359b72c681a2c32eaa7ace8d346b7e8ce09d172a683acf6853161d9c", - "sha256:b44624a38d07d3c954c84ad302c29f7930f4bf01443beef5589e9157b14e2a29", - "sha256:be42a601aaaeb7a317f818490a39d153952a97c40c6e9beeb2a1103616405348", - "sha256:eee4d94b1a626490fc8170ffd788883f8c641b576e11ba9b4a29c9f6623371e0", - "sha256:f69d1201a4750f763971ea8364ed95ee888fc128968b39d38883a72a4d005895" - ], - "version": "==2018.2.21" - }, - "requests": { - "hashes": [ - "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", - "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" - ], - "version": "==2.18.4" - }, - "requests-toolbelt": { - "hashes": [ - "sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237", - "sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5" - ], - "version": "==0.8.0" - }, - "ruamel.yaml": { - "hashes": [ - "sha256:01e30ecb1b1c0ebf9fce814dc20dace402571517277799291202b61b22096c24", - "sha256:02babffd019911841ba01b76e23dfec7c9e9b2725503fb2698c4982fa1a6e835", - "sha256:072f6364a89972e8dc0afdce3335a709d5464dfeaa4f736d092a54574338b874", - "sha256:14d161558e3bf89e87d77c218098be22fa9a0d6d0bea40250fce525b1d0cbee2", - "sha256:5504398fc755a2b14c9983b2101161a8591a4b30812590cc1c365e7fcc117dfa", - "sha256:68c8f2986bcb91b6db1aea8698941769840c7257e951a9377048f7eff35be773", - "sha256:6d05c5a5baf829c70916c226ef3200650846a7227de226bca8a59efaf88bb973", - "sha256:6d7929b24e329d662fa43b657fddfee5260e2d35d0a543065cd755d4e17a9b2f", - "sha256:8dc74821e4bb6b21fb1ab35964e159391d99ee44981d07d57bf96e2395f3ef75", - "sha256:9225c83952d28f302cfc23c3d9a6f8231bfd581476d7aff1e3c7de49eecb4ee9", - "sha256:b6c5d5f03ba78e3f27c7188a00c4e09b6a4507fe3154ba40a294e09cb30ee016", - "sha256:c0908896e34b617ead40552cab03c1769bdc43d1da02419160dc900c5dfddde2", - "sha256:c41e04b526d0153c9246cfab87d7ddefdc9f165cb8886a8ec48ba7a2b73069f6", - "sha256:e2d2715bf92156bec5fb42e92e95dac1c4d9904f8a3d4e2d0c438758fe9092d7", - "sha256:e3bbfe0d294e08fdbb0cb05485435a2ceb4e168e98b5dc611f051c1864986b4b", - "sha256:f2d02a4af5a13b09d0b823cdd0317b54f3e0115e50b5ac4d9840c3a1b566817f", - "sha256:fcfc24a21594c071cc4588e84b7657a1f47ebcf6037c6c43fa15c4bbd3989ec2" - ], - "version": "==0.15.35" - }, - "six": { - "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" - ], - "version": "==1.11.0" - }, - "tqdm": { - "hashes": [ - "sha256:5ec0d4442358e55cdb4a0471d04c6c831518fd8837f259db5537d90feab380df", - "sha256:f66468c14ccd011a627734c9b3fd72f20ce16f8faecc47384eb2507af5924fb9" - ], - "version": "==4.19.6" - }, - "twine": { - "hashes": [ - "sha256:caa45b7987fc96321258cd7668e3be2ff34064f5c66d2d975b641adca659c1ab", - "sha256:d3ce5c480c22ccfb761cd358526e862b32546d2fe4bc93d46b5cf04ea3cc46ca" - ], - "version": "==1.9.1" - }, - "tzlocal": { - "hashes": [ - "sha256:4ebeb848845ac898da6519b9b31879cf13b6626f7184c496037b818e238f2c4e" - ], - "version": "==1.5.1" - }, - "urllib3": { - "hashes": [ - "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", - "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" - ], - "version": "==1.22" - }, - "werkzeug": { - "hashes": [ - "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", - "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" - ], - "version": "==0.14.1" - }, - "whitenoise": { - "hashes": [ - "sha256:15f43b2e701821b95c9016cf469d29e2a546cb1c7dead584ba82c36f843995cf", - "sha256:9d81515f2b5b27051910996e1e860b1332e354d9e7bcf30c98f21dcb6713e0dd" - ], - "version": "==3.3.1" - } - } -} From b7564057f7676e7bfa25215dc715166b5dc60047 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 6 Nov 2018 16:13:39 +0900 Subject: [PATCH 083/218] Format --- pipenv/vendor/vistir/path.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index ba008159..b1236884 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -8,7 +8,6 @@ import os import posixpath import shutil import stat -import sys import warnings import six @@ -206,9 +205,8 @@ def mkdir_p(newdir, mode=0o777): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): raise OSError( - "A file with the same name as the desired dir, '{0}', already exists.".format( - to_text(newdir, encoding="utf-8") - ) + "A file with the same name as the desired dir, '{0}', " + "already exists.".format(to_text(newdir, encoding="utf-8")) ) os.makedirs(os.path.join(head, tail), mode) From 3dcd206068c47201cd54f61f30a2875faa046438 Mon Sep 17 00:00:00 2001 From: frostming Date: Tue, 6 Nov 2018 16:05:13 +0800 Subject: [PATCH 084/218] Fix non-ASCII support for prettytoml --- news/2737.bugfix.rst | 1 + pipenv/patched/prettytoml/tokens/py2toml.py | 29 +++- pipenv/patched/prettytoml/tokens/toml2py.py | 70 +++++---- .../patches/patched/prettytoml-unicode.patch | 140 ++++++++++++++++++ tests/unit/test_vendor.py | 8 + 5 files changed, 208 insertions(+), 40 deletions(-) create mode 100644 news/2737.bugfix.rst create mode 100644 tasks/vendoring/patches/patched/prettytoml-unicode.patch diff --git a/news/2737.bugfix.rst b/news/2737.bugfix.rst new file mode 100644 index 00000000..bddcff91 --- /dev/null +++ b/news/2737.bugfix.rst @@ -0,0 +1 @@ +Handle non-ASCII characters correctly in TOML. diff --git a/pipenv/patched/prettytoml/tokens/py2toml.py b/pipenv/patched/prettytoml/tokens/py2toml.py index 82991958..2decd021 100644 --- a/pipenv/patched/prettytoml/tokens/py2toml.py +++ b/pipenv/patched/prettytoml/tokens/py2toml.py @@ -2,6 +2,7 @@ """ A converter of python values to TOML Token instances. """ +from __future__ import unicode_literals import codecs import datetime import six @@ -81,10 +82,30 @@ def create_string_token(text, bare_string_allowed=False, multiline_strings_allow def _escape_single_line_quoted_string(text): - if six.PY2: - return text.encode('unicode-escape').encode('string-escape').replace('"', '\\"').replace("\\'", "'") - else: - return codecs.encode(text, 'unicode-escape').decode().replace('"', '\\"') + text = text.decode('utf-8') if isinstance(text, six.binary_type) else text + start = 0 + i = 0 + res = [] + _escapes = {'\n': '\\n', '\r': '\\r', '\\': '\\\\', '\t': '\\t', + '\b': '\\b', '\f': '\\f', '"': '\\"'} + + def flush(): + if start < i: + res.append(text[start:i]) + return i + 1 + + while i < len(text): + c = text[i] + if c in _escapes: + start = flush() + res.append(_escapes[c]) + elif ord(c) < 0x20: + start = flush() + res.append('\\u%04x' % ord(c)) + i += 1 + + flush() + return ''.join(res) def _create_multiline_string_token(text): diff --git a/pipenv/patched/prettytoml/tokens/toml2py.py b/pipenv/patched/prettytoml/tokens/toml2py.py index 2bf9c1c2..98186ca5 100644 --- a/pipenv/patched/prettytoml/tokens/toml2py.py +++ b/pipenv/patched/prettytoml/tokens/toml2py.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import re import string import iso8601 @@ -19,7 +20,7 @@ def deserialize(token): Raises DeserializationError when appropriate. """ - + if token.type == TYPE_BOOLEAN: return _to_boolean(token) elif token.type == TYPE_INTEGER: @@ -39,42 +40,39 @@ def _unescape_str(text): """ Unescapes a string according the TOML spec. Raises BadEscapeCharacter when appropriate. """ - - # Detect bad escape jobs - bad_escape_regexp = re.compile(r'([^\\]|^)\\[^btnfr"\\uU]') - if bad_escape_regexp.findall(text): - raise BadEscapeCharacter - - # Do the unescaping - if six.PY2: - return _unicode_escaped_string(text).decode('string-escape').decode('unicode-escape') - else: - return codecs.decode(_unicode_escaped_string(text), 'unicode-escape') - - -def _unicode_escaped_string(text): - """ - Escapes all unicode characters in the given string - """ - - if six.PY2: - text = unicode(text) - - def is_unicode(c): - return c.lower() not in string.ascii_letters + string.whitespace + string.punctuation + string.digits - - def escape_unicode_char(x): - if six.PY2: - return x.encode('unicode-escape') + tokens = [] + i = 0 + basicstr_re = re.compile(r'[^"\\\000-\037]*') + unicode_re = re.compile(r'[uU]((?<=u)[a-fA-F0-9]{4}|(?<=U)[a-fA-F0-9]{8})') + escapes = { + 'b': '\b', + 't': '\t', + 'n': '\n', + 'f': '\f', + 'r': '\r', + '\\': '\\', + '"': '"', + '/': '/', + "'": "'" + } + while True: + m = basicstr_re.match(text, i) + i = m.end() + tokens.append(m.group()) + if i == len(text) or text[i] != '\\': + break else: - return codecs.encode(x, 'unicode-escape') - - if any(is_unicode(c) for c in text): - homogeneous_chars = tuple(escape_unicode_char(c) if is_unicode(c) else c.encode() for c in text) - homogeneous_bytes = functools.reduce(operator.add, homogeneous_chars) - return homogeneous_bytes.decode() - else: - return text + i += 1 + if unicode_re.match(text, i): + m = unicode_re.match(text, i) + i = m.end() + tokens.append(six.unichr(int(m.group(1), 16))) + else: + if text[i] not in escapes: + raise BadEscapeCharacter + tokens.append(escapes[text[i]]) + i += 1 + return ''.join(tokens) def _to_string(token): diff --git a/tasks/vendoring/patches/patched/prettytoml-unicode.patch b/tasks/vendoring/patches/patched/prettytoml-unicode.patch new file mode 100644 index 00000000..6b9ffecf --- /dev/null +++ b/tasks/vendoring/patches/patched/prettytoml-unicode.patch @@ -0,0 +1,140 @@ +diff --git a/pipenv/patched/prettytoml/tokens/py2toml.py b/pipenv/patched/prettytoml/tokens/py2toml.py +index 8299195..2decd02 100644 +--- a/pipenv/patched/prettytoml/tokens/py2toml.py ++++ b/pipenv/patched/prettytoml/tokens/py2toml.py +@@ -2,6 +2,7 @@ + """ + A converter of python values to TOML Token instances. + """ ++from __future__ import unicode_literals + import codecs + import datetime + import six +@@ -81,10 +82,30 @@ def create_string_token(text, bare_string_allowed=False, multiline_strings_allow + + + def _escape_single_line_quoted_string(text): +- if six.PY2: +- return text.encode('unicode-escape').encode('string-escape').replace('"', '\\"').replace("\\'", "'") +- else: +- return codecs.encode(text, 'unicode-escape').decode().replace('"', '\\"') ++ text = text.decode('utf-8') if isinstance(text, six.binary_type) else text ++ start = 0 ++ i = 0 ++ res = [] ++ _escapes = {'\n': '\\n', '\r': '\\r', '\\': '\\\\', '\t': '\\t', ++ '\b': '\\b', '\f': '\\f', '"': '\\"'} ++ ++ def flush(): ++ if start < i: ++ res.append(text[start:i]) ++ return i + 1 ++ ++ while i < len(text): ++ c = text[i] ++ if c in _escapes: ++ start = flush() ++ res.append(_escapes[c]) ++ elif ord(c) < 0x20: ++ start = flush() ++ res.append('\\u%04x' % ord(c)) ++ i += 1 ++ ++ flush() ++ return ''.join(res) + + + def _create_multiline_string_token(text): +diff --git a/pipenv/patched/prettytoml/tokens/toml2py.py b/pipenv/patched/prettytoml/tokens/toml2py.py +index 2bf9c1c..98186ca 100644 +--- a/pipenv/patched/prettytoml/tokens/toml2py.py ++++ b/pipenv/patched/prettytoml/tokens/toml2py.py +@@ -1,3 +1,4 @@ ++from __future__ import unicode_literals + import re + import string + import iso8601 +@@ -19,7 +20,7 @@ def deserialize(token): + + Raises DeserializationError when appropriate. + """ +- ++ + if token.type == TYPE_BOOLEAN: + return _to_boolean(token) + elif token.type == TYPE_INTEGER: +@@ -39,42 +40,39 @@ def _unescape_str(text): + """ + Unescapes a string according the TOML spec. Raises BadEscapeCharacter when appropriate. + """ +- +- # Detect bad escape jobs +- bad_escape_regexp = re.compile(r'([^\\]|^)\\[^btnfr"\\uU]') +- if bad_escape_regexp.findall(text): +- raise BadEscapeCharacter +- +- # Do the unescaping +- if six.PY2: +- return _unicode_escaped_string(text).decode('string-escape').decode('unicode-escape') +- else: +- return codecs.decode(_unicode_escaped_string(text), 'unicode-escape') +- +- +-def _unicode_escaped_string(text): +- """ +- Escapes all unicode characters in the given string +- """ +- +- if six.PY2: +- text = unicode(text) +- +- def is_unicode(c): +- return c.lower() not in string.ascii_letters + string.whitespace + string.punctuation + string.digits +- +- def escape_unicode_char(x): +- if six.PY2: +- return x.encode('unicode-escape') ++ tokens = [] ++ i = 0 ++ basicstr_re = re.compile(r'[^"\\\000-\037]*') ++ unicode_re = re.compile(r'[uU]((?<=u)[a-fA-F0-9]{4}|(?<=U)[a-fA-F0-9]{8})') ++ escapes = { ++ 'b': '\b', ++ 't': '\t', ++ 'n': '\n', ++ 'f': '\f', ++ 'r': '\r', ++ '\\': '\\', ++ '"': '"', ++ '/': '/', ++ "'": "'" ++ } ++ while True: ++ m = basicstr_re.match(text, i) ++ i = m.end() ++ tokens.append(m.group()) ++ if i == len(text) or text[i] != '\\': ++ break + else: +- return codecs.encode(x, 'unicode-escape') +- +- if any(is_unicode(c) for c in text): +- homogeneous_chars = tuple(escape_unicode_char(c) if is_unicode(c) else c.encode() for c in text) +- homogeneous_bytes = functools.reduce(operator.add, homogeneous_chars) +- return homogeneous_bytes.decode() +- else: +- return text ++ i += 1 ++ if unicode_re.match(text, i): ++ m = unicode_re.match(text, i) ++ i = m.end() ++ tokens.append(six.unichr(int(m.group(1), 16))) ++ else: ++ if text[i] not in escapes: ++ raise BadEscapeCharacter ++ tokens.append(escapes[text[i]]) ++ i += 1 ++ return ''.join(tokens) + + + def _to_string(token): diff --git a/tests/unit/test_vendor.py b/tests/unit/test_vendor.py index 915af863..1894b6ff 100644 --- a/tests/unit/test_vendor.py +++ b/tests/unit/test_vendor.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # We need to import the patched packages directly from sys.path, so the # identity checks can pass. import pipenv # noqa @@ -8,6 +9,7 @@ import os import pytest import pytz +import contoml from pipfile.api import PipfileParser from prettytoml import lexer, tokens from prettytoml.elements.atomic import AtomicElement @@ -104,3 +106,9 @@ class TestPipfileParser: def test_token_date(dt, content): token = create_primitive_token(dt) assert token == tokens.Token(tokens.TYPE_DATE, content) + + +def test_dump_nonascii_string(): + content = 'name = "Stažené"\n' + toml_content = contoml.dumps(contoml.loads(content)) + assert toml_content == content From 4972248e3370c6d56d75ff081a329cfdfe172a8b Mon Sep 17 00:00:00 2001 From: frostming Date: Tue, 6 Nov 2018 16:56:27 +0800 Subject: [PATCH 085/218] fix unicode for py2 --- pipenv/patched/prettytoml/tokens/toml2py.py | 1 + .../patches/patched/prettytoml-unicode.patch | 23 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pipenv/patched/prettytoml/tokens/toml2py.py b/pipenv/patched/prettytoml/tokens/toml2py.py index 98186ca5..56804437 100644 --- a/pipenv/patched/prettytoml/tokens/toml2py.py +++ b/pipenv/patched/prettytoml/tokens/toml2py.py @@ -40,6 +40,7 @@ def _unescape_str(text): """ Unescapes a string according the TOML spec. Raises BadEscapeCharacter when appropriate. """ + text = text.decode('utf-8') if isinstance(text, six.binary_type) else text tokens = [] i = 0 basicstr_re = re.compile(r'[^"\\\000-\037]*') diff --git a/tasks/vendoring/patches/patched/prettytoml-unicode.patch b/tasks/vendoring/patches/patched/prettytoml-unicode.patch index 6b9ffecf..4001b285 100644 --- a/tasks/vendoring/patches/patched/prettytoml-unicode.patch +++ b/tasks/vendoring/patches/patched/prettytoml-unicode.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/patched/prettytoml/tokens/py2toml.py b/pipenv/patched/prettytoml/tokens/py2toml.py -index 8299195..2decd02 100644 +index 82991958..2decd021 100644 --- a/pipenv/patched/prettytoml/tokens/py2toml.py +++ b/pipenv/patched/prettytoml/tokens/py2toml.py @@ -2,6 +2,7 @@ @@ -11,8 +11,8 @@ index 8299195..2decd02 100644 import datetime import six @@ -81,10 +82,30 @@ def create_string_token(text, bare_string_allowed=False, multiline_strings_allow - - + + def _escape_single_line_quoted_string(text): - if six.PY2: - return text.encode('unicode-escape').encode('string-escape').replace('"', '\\"').replace("\\'", "'") @@ -42,11 +42,11 @@ index 8299195..2decd02 100644 + + flush() + return ''.join(res) - - + + def _create_multiline_string_token(text): diff --git a/pipenv/patched/prettytoml/tokens/toml2py.py b/pipenv/patched/prettytoml/tokens/toml2py.py -index 2bf9c1c..98186ca 100644 +index 2bf9c1c2..56804437 100644 --- a/pipenv/patched/prettytoml/tokens/toml2py.py +++ b/pipenv/patched/prettytoml/tokens/toml2py.py @@ -1,3 +1,4 @@ @@ -55,15 +55,15 @@ index 2bf9c1c..98186ca 100644 import string import iso8601 @@ -19,7 +20,7 @@ def deserialize(token): - + Raises DeserializationError when appropriate. """ -- +- + if token.type == TYPE_BOOLEAN: return _to_boolean(token) elif token.type == TYPE_INTEGER: -@@ -39,42 +40,39 @@ def _unescape_str(text): +@@ -39,42 +40,40 @@ def _unescape_str(text): """ Unescapes a string according the TOML spec. Raises BadEscapeCharacter when appropriate. """ @@ -94,6 +94,7 @@ index 2bf9c1c..98186ca 100644 - def escape_unicode_char(x): - if six.PY2: - return x.encode('unicode-escape') ++ text = text.decode('utf-8') if isinstance(text, six.binary_type) else text + tokens = [] + i = 0 + basicstr_re = re.compile(r'[^"\\\000-\037]*') @@ -135,6 +136,6 @@ index 2bf9c1c..98186ca 100644 + tokens.append(escapes[text[i]]) + i += 1 + return ''.join(tokens) - - + + def _to_string(token): From 524e31ff5a704fea576785f9ef823e969fcff62e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 6 Nov 2018 04:35:15 -0500 Subject: [PATCH 086/218] Switch to tomlkit for parsing and writing - Update tomlkit to preserve inline comments when deleting elemeents Signed-off-by: Dan Ryan --- pipenv/project.py | 58 ++++++++++++++----- pipenv/vendor/tomlkit/items.py | 12 ++++ .../patches/vendor/tomlkit-update-items.patch | 23 ++++++++ 3 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 tasks/vendoring/patches/vendor/tomlkit-update-items.patch diff --git a/pipenv/project.py b/pipenv/project.py index a60271e9..1354eff0 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -17,6 +17,7 @@ import pipfile.api import six import vistir import toml +import tomlkit from .cmdparse import Script from .utils import ( @@ -154,6 +155,9 @@ class Project(object): self._requirements_location = None self._original_dir = os.path.abspath(os.curdir) self._which = which + self._build_system = { + "requires": ["setuptools", "wheel"] + } self.python_version = python_version # Hack to skip this during pipenv run, or -r. if ("run" not in sys.argv) and chdir: @@ -574,32 +578,55 @@ class Project(object): _pipfile_cache.clear() def _parse_pipfile(self, contents): + toml_encoder = toml.TomlPreserveInlineDictEncoder() + toml_encoder.get_empty_table() # If any outline tables are present... if ("[packages." in contents) or ("[dev-packages." in contents): - data = toml.loads(contents) + data = tomlkit.parse(contents) # Convert all outline tables to inline tables. for section in ("packages", "dev-packages"): for package in data.get(section, {}): # Convert things to inline tables — fancy :) if hasattr(data[section][package], "keys"): + table = tomlkit.inline_table() _data = data[section][package] - data[section][package] = toml.TomlDecoder().get_empty_inline_table() - data[section][package].update(_data) - toml_encoder = toml.TomlEncoder(preserve=True) + table.update(_data) + data[section][package] = table # We lose comments here, but it's for the best.) try: - return contoml.loads(toml.dumps(data, encoder=toml_encoder)) + return data except RuntimeError: - return toml.loads(toml.dumps(data, encoder=toml_encoder)) + return toml.loads(tomlkit.dumps(data, encoder=toml_encoder)) else: # Fallback to toml parser, for large files. try: - return contoml.loads(contents) + return tomlkit.loads(contents) except Exception: - return toml.loads(contents) + return toml.loads(contents, encoder=toml_encoder) + + def _read_pyproject(self): + pyproject = self.path_to("pyproject.toml") + if os.path.exists(pyproject): + self._pyproject = toml.load(pyproject) + build_system = self._pyproject.get("build-system", None) + if not os.path.exists(self.path_to("setup.py")): + if not build_system or not build_system.get("requires"): + build_system = { + "requires": ["setuptools>=38.2.5", "wheel"], + "build-backend": "setuptools.build_meta", + } + self._build_system = build_system + + @property + def build_requires(self): + return self._build_system.get("requires", []) + + @property + def build_backend(self): + return self._build_system.get("build-backend", None) @property def settings(self): @@ -833,16 +860,21 @@ class Project(object): if path is None: path = self.pipfile_location try: - formatted_data = contoml.dumps(data).rstrip() + formatted_data = tomlkit.dumps(data).rstrip() except Exception: + document = tomlkit.document() for section in ("packages", "dev-packages"): + document[section] = tomlkit.container.Table() for package in data.get(section, {}): # Convert things to inline tables — fancy :) - if hasattr(data[section][package], "keys"): _data = data[section][package] - data[section][package] = toml.TomlDecoder().get_empty_inline_table() - data[section][package].update(_data) - formatted_data = toml.dumps(data).rstrip() + if hasattr(_data, "keys"): + table = tomlkit.inline_table() + table.update(data) + document[section][package] = table + else: + document[section][package] = tomlkit.string(_data) + formatted_data = tomlkit.dumps(document).rstrip() if ( vistir.compat.Path(path).absolute() diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 781e2e98..5b2fd15f 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -789,6 +789,18 @@ class Table(Item, dict): return self def remove(self, key): # type: (Union[Key, str]) -> Table + data = [ + (k, v) for k, v in self._value.body + if getattr(k, "key", "") == getattr(key, "key", key) + and getattr(v, "trivia", None) + ] + indexed_keys = [(i, k) for i, k in enumerate(self.keys())] + target_idx = next(iter(i for i, k in indexed_keys if k == data[0][0].key), None) + if target_idx == 0: + self.comment(data[0][1].trivia.comment) + else: + target_key = self._value.body[target_idx - 1][0].key + self._value[target_key].comment(data[0][1].trivia.comment) self._value.remove(key) if isinstance(key, Key): diff --git a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch new file mode 100644 index 00000000..316a548d --- /dev/null +++ b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch @@ -0,0 +1,23 @@ +diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py +index 781e2e98..5b2fd15f 100644 +--- a/pipenv/vendor/tomlkit/items.py ++++ b/pipenv/vendor/tomlkit/items.py +@@ -789,6 +789,18 @@ class Table(Item, dict): + return self + + def remove(self, key): # type: (Union[Key, str]) -> Table ++ data = [ ++ (k, v) for k, v in self._value.body ++ if getattr(k, "key", "") == getattr(key, "key", key) ++ and getattr(v, "trivia", None) ++ ] ++ indexed_keys = [(i, k) for i, k in enumerate(self.keys())] ++ target_idx = next(iter(i for i, k in indexed_keys if k == data[0][0].key), None) ++ if target_idx == 0: ++ self.comment(data[0][1].trivia.comment) ++ else: ++ target_key = self._value.body[target_idx - 1][0].key ++ self._value[target_key].comment(data[0][1].trivia.comment) + self._value.remove(key) + + if isinstance(key, Key): From 4c542938e36e711022950cf467b02b677887cb95 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 6 Nov 2018 04:56:11 -0500 Subject: [PATCH 087/218] Fix indentation Signed-off-by: Dan Ryan --- pipenv/project.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 1354eff0..f1366472 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -865,15 +865,14 @@ class Project(object): document = tomlkit.document() for section in ("packages", "dev-packages"): document[section] = tomlkit.container.Table() + # Convert things to inline tables — fancy :) for package in data.get(section, {}): - # Convert things to inline tables — fancy :) - _data = data[section][package] - if hasattr(_data, "keys"): + if hasattr(data[section][package], "keys"): table = tomlkit.inline_table() - table.update(data) + table.update(data[section][package]) document[section][package] = table else: - document[section][package] = tomlkit.string(_data) + document[section][package] = tomlkit.string(data[section][package]) formatted_data = tomlkit.dumps(document).rstrip() if ( From 7b18c124488d0f9fbb652ca0e14d95b03b36b1e2 Mon Sep 17 00:00:00 2001 From: jxltom Date: Tue, 6 Nov 2018 17:55:54 +0800 Subject: [PATCH 088/218] Add explicit message if there is nothing to be removed during uninstall --- pipenv/core.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 1922b28f..8636dfd4 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1154,16 +1154,17 @@ def do_purge(bare=False, downloads=False, allow_global=False): # Remove setuptools, pip, etc from targets for removal to_remove = installed - bad_pkgs + # Skip purging if there is no packages which needs to be removed + if not to_remove: + if not bare: + click.echo("Found 0 installed package, skip purging.") + click.echo(crayons.green("Environment now purged and fresh!")) + return installed + if not bare: click.echo( fix_utf8("Found {0} installed package(s), purging…".format(len(to_remove))) ) - - # Skip purging if there is no packages which needs to be removed - if not to_remove: - if not bare: - click.echo(crayons.green("Environment now purged and fresh!")) - return installed command = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=allow_global)), From e98e7529d420c610daf3fde54be7180df4f6de19 Mon Sep 17 00:00:00 2001 From: jxltom Date: Tue, 6 Nov 2018 22:38:21 +0800 Subject: [PATCH 089/218] Remove u'' or '' in output of pipenv uninstall and clean --- pipenv/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index fb68fcb1..6f7812bd 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2087,7 +2087,7 @@ def do_uninstall( normalized_bad_pkg = canonicalize_name(bad_package) if normalized_bad_pkg in package_map: if environments.is_verbose(): - click.echo("Ignoring {0}.".format(repr(bad_package)), err=True) + click.echo("Ignoring {0}.".format(bad_package), err=True) pkg_name_index = package_names.index(package_map[normalized_bad_pkg]) del package_names[pkg_name_index] used_packages = develop | default & installed_package_names @@ -2126,7 +2126,7 @@ def do_uninstall( for normalized, package_name in selected_pkg_map.items(): click.echo( crayons.white( - fix_utf8("Uninstalling {0}…".format(repr(package_name))), bold=True + fix_utf8("Uninstalling {0}…".format(package_name)), bold=True ) ) # Uninstall the package. @@ -2643,7 +2643,7 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro for bad_package in BAD_PACKAGES: if canonicalize_name(bad_package) in installed_package_names: if environments.is_verbose(): - click.echo("Ignoring {0}.".format(repr(bad_package)), err=True) + click.echo("Ignoring {0}.".format(bad_package), err=True) del installed_package_names[installed_package_names.index( canonicalize_name(bad_package) )] @@ -2663,7 +2663,7 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro if not bare: click.echo( crayons.white( - fix_utf8("Uninstalling {0}…".format(repr(apparent_bad_package))), bold=True + fix_utf8("Uninstalling {0}…".format(apparent_bad_package)), bold=True ) ) # Uninstall the package. From 5d57ec738a55ed791626c9096a0e43db0a822b07 Mon Sep 17 00:00:00 2001 From: jxltom Date: Tue, 6 Nov 2018 22:44:48 +0800 Subject: [PATCH 090/218] Add trivial news for issue 3179 --- news/3179.trivial | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/3179.trivial diff --git a/news/3179.trivial b/news/3179.trivial new file mode 100644 index 00000000..4b9059f7 --- /dev/null +++ b/news/3179.trivial @@ -0,0 +1 @@ +Better output of pipenv uninstall and clean From 8a657dc7bc4353490b5d34e1936195757f718293 Mon Sep 17 00:00:00 2001 From: jxltom Date: Tue, 6 Nov 2018 23:29:29 +0800 Subject: [PATCH 091/218] Do not add index source per-line in requirements files --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index fb68fcb1..5b532300 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -726,7 +726,7 @@ def do_install_dependencies( index_args = prepare_pip_source_args(project.sources) index_args = " ".join(index_args).replace(" -", "\n-") deps = [ - req.as_line(sources=project.sources, include_hashes=False) for req in deps_list + req.as_line(sources=False, include_hashes=False) for req in deps_list ] # Output only default dependencies click.echo(index_args) From 722e33c77ab4b113b68dccc6851ad5722e4ab11b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 6 Nov 2018 18:09:09 -0500 Subject: [PATCH 092/218] Fix tomlkit updates Signed-off-by: Dan Ryan --- pipenv/vendor/tomlkit/container.py | 4 +++ pipenv/vendor/tomlkit/items.py | 12 ------- .../patches/vendor/tomlkit-update-items.patch | 34 +++++++------------ 3 files changed, 17 insertions(+), 33 deletions(-) diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index 37014921..729083a0 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -189,6 +189,10 @@ class Container(dict): if idx is None: raise NonExistentKey(key) + old_data = self._body[idx][1] + trivia = getattr(old_data, "trivia", None) + if trivia and idx > 0 and getattr(trivia, "comment", None): + self._body[idx - 1][1].comment(trivia.comment) self._body[idx] = (None, Null()) super(Container, self).__delitem__(key.key) diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 5b2fd15f..781e2e98 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -789,18 +789,6 @@ class Table(Item, dict): return self def remove(self, key): # type: (Union[Key, str]) -> Table - data = [ - (k, v) for k, v in self._value.body - if getattr(k, "key", "") == getattr(key, "key", key) - and getattr(v, "trivia", None) - ] - indexed_keys = [(i, k) for i, k in enumerate(self.keys())] - target_idx = next(iter(i for i, k in indexed_keys if k == data[0][0].key), None) - if target_idx == 0: - self.comment(data[0][1].trivia.comment) - else: - target_key = self._value.body[target_idx - 1][0].key - self._value[target_key].comment(data[0][1].trivia.comment) self._value.remove(key) if isinstance(key, Key): diff --git a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch index 316a548d..304b4375 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch @@ -1,23 +1,15 @@ -diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py -index 781e2e98..5b2fd15f 100644 ---- a/pipenv/vendor/tomlkit/items.py -+++ b/pipenv/vendor/tomlkit/items.py -@@ -789,6 +789,18 @@ class Table(Item, dict): - return self +diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py +index 37014921..729083a0 100644 +--- a/pipenv/vendor/tomlkit/container.py ++++ b/pipenv/vendor/tomlkit/container.py +@@ -189,6 +189,10 @@ class Container(dict): + if idx is None: + raise NonExistentKey(key) - def remove(self, key): # type: (Union[Key, str]) -> Table -+ data = [ -+ (k, v) for k, v in self._value.body -+ if getattr(k, "key", "") == getattr(key, "key", key) -+ and getattr(v, "trivia", None) -+ ] -+ indexed_keys = [(i, k) for i, k in enumerate(self.keys())] -+ target_idx = next(iter(i for i, k in indexed_keys if k == data[0][0].key), None) -+ if target_idx == 0: -+ self.comment(data[0][1].trivia.comment) -+ else: -+ target_key = self._value.body[target_idx - 1][0].key -+ self._value[target_key].comment(data[0][1].trivia.comment) - self._value.remove(key) ++ old_data = self._body[idx][1] ++ trivia = getattr(old_data, "trivia", None) ++ if trivia and idx > 0 and getattr(trivia, "comment", None): ++ self._body[idx - 1][1].comment(trivia.comment) + self._body[idx] = (None, Null()) - if isinstance(key, Key): + super(Container, self).__delitem__(key.key) From 4610043c64d4eeb6064b03761d7b031053f68a07 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 6 Nov 2018 18:10:21 -0500 Subject: [PATCH 093/218] Update pipfile toml implementation Signed-off-by: Dan Ryan --- pipenv/project.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index f1366472..d9ba5f6d 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -578,9 +578,8 @@ class Project(object): _pipfile_cache.clear() def _parse_pipfile(self, contents): - toml_encoder = toml.TomlPreserveInlineDictEncoder() - toml_encoder.get_empty_table() # If any outline tables are present... + toml_encoder = toml.TomlPreserveInlineDictEncoder() if ("[packages." in contents) or ("[dev-packages." in contents): data = tomlkit.parse(contents) # Convert all outline tables to inline tables. @@ -605,6 +604,7 @@ class Project(object): return tomlkit.loads(contents) except Exception: + toml_encoder.get_empty_table() return toml.loads(contents, encoder=toml_encoder) def _read_pyproject(self): @@ -972,7 +972,10 @@ class Project(object): name = self.get_package_name_in_pipfile(package_name, dev) key = "dev-packages" if dev else "packages" p = self.parsed_pipfile - lines = [l for l in p[key].serialized().splitlines()] + try: + lines = [l for l in p.item(key).as_string().splitlines()] + except AttributeError: + lines = [l for l in p[key].serialized().splitlines()] if not any(line.startswith("#") for line in lines) and name: del p[key][name] self.write_toml(p) @@ -989,10 +992,16 @@ class Project(object): has_comments_as_lines = False for section in ("dev-packages", "packages"): pipfile_section = self.parsed_pipfile.get(section, {}) - lines = [ - l for l in parsed[section].serialized().splitlines() - if section in parsed.keys() - ] + try: + lines = [ + l for l in p.item(section).as_string().splitlines() + if section in parsed.keys() + ] + except AttributeError: + lines = [ + l for l in p[section].serialized().splitlines() + if section in parsed.keys() + ] pipfile_packages = [ pkg_name for pkg_name in pipfile_section.keys() if pep423_name(pkg_name) in packages From 67334d001f4286764bc4eca7193aedaafe7d8e4e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 6 Nov 2018 20:19:45 -0500 Subject: [PATCH 094/218] Fix toml fallback Signed-off-by: Dan Ryan --- pipenv/project.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index d9ba5f6d..da6d8560 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -579,7 +579,6 @@ class Project(object): def _parse_pipfile(self, contents): # If any outline tables are present... - toml_encoder = toml.TomlPreserveInlineDictEncoder() if ("[packages." in contents) or ("[dev-packages." in contents): data = tomlkit.parse(contents) # Convert all outline tables to inline tables. @@ -592,20 +591,16 @@ class Project(object): table.update(_data) data[section][package] = table # We lose comments here, but it's for the best.) - try: - return data - - except RuntimeError: - return toml.loads(tomlkit.dumps(data, encoder=toml_encoder)) + return data else: - # Fallback to toml parser, for large files. try: - return tomlkit.loads(contents) + return tomlkit.parse(contents) + # Fallback to toml parser, for large files. except Exception: - toml_encoder.get_empty_table() - return toml.loads(contents, encoder=toml_encoder) + toml_decoder = toml.decoder.TomlDecoder() + return toml.loads(contents, encoder=toml_decoder) def _read_pyproject(self): pyproject = self.path_to("pyproject.toml") From c47dd0134eb7bffb5166cacdee092d33d3d6722f Mon Sep 17 00:00:00 2001 From: frostming Date: Wed, 7 Nov 2018 09:33:39 +0800 Subject: [PATCH 095/218] fix patch file --- .../patches/patched/prettytoml-unicode.patch | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tasks/vendoring/patches/patched/prettytoml-unicode.patch b/tasks/vendoring/patches/patched/prettytoml-unicode.patch index 4001b285..3bf3ab56 100644 --- a/tasks/vendoring/patches/patched/prettytoml-unicode.patch +++ b/tasks/vendoring/patches/patched/prettytoml-unicode.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/patched/prettytoml/tokens/py2toml.py b/pipenv/patched/prettytoml/tokens/py2toml.py -index 82991958..2decd021 100644 +index 8299195..2decd02 100644 --- a/pipenv/patched/prettytoml/tokens/py2toml.py +++ b/pipenv/patched/prettytoml/tokens/py2toml.py @@ -2,6 +2,7 @@ @@ -11,8 +11,8 @@ index 82991958..2decd021 100644 import datetime import six @@ -81,10 +82,30 @@ def create_string_token(text, bare_string_allowed=False, multiline_strings_allow - - + + def _escape_single_line_quoted_string(text): - if six.PY2: - return text.encode('unicode-escape').encode('string-escape').replace('"', '\\"').replace("\\'", "'") @@ -42,11 +42,11 @@ index 82991958..2decd021 100644 + + flush() + return ''.join(res) - - + + def _create_multiline_string_token(text): diff --git a/pipenv/patched/prettytoml/tokens/toml2py.py b/pipenv/patched/prettytoml/tokens/toml2py.py -index 2bf9c1c2..56804437 100644 +index 2bf9c1c..5680443 100644 --- a/pipenv/patched/prettytoml/tokens/toml2py.py +++ b/pipenv/patched/prettytoml/tokens/toml2py.py @@ -1,3 +1,4 @@ @@ -55,10 +55,10 @@ index 2bf9c1c2..56804437 100644 import string import iso8601 @@ -19,7 +20,7 @@ def deserialize(token): - + Raises DeserializationError when appropriate. """ -- +- + if token.type == TYPE_BOOLEAN: return _to_boolean(token) @@ -136,6 +136,6 @@ index 2bf9c1c2..56804437 100644 + tokens.append(escapes[text[i]]) + i += 1 + return ''.join(tokens) - - + + def _to_string(token): From 4310bcaf2353f11760efd9f59375e95fcb39cd53 Mon Sep 17 00:00:00 2001 From: frostming Date: Wed, 7 Nov 2018 10:06:14 +0800 Subject: [PATCH 096/218] remove the tricky lines --- .../patches/patched/prettytoml-unicode.patch | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/tasks/vendoring/patches/patched/prettytoml-unicode.patch b/tasks/vendoring/patches/patched/prettytoml-unicode.patch index 3bf3ab56..54f4c621 100644 --- a/tasks/vendoring/patches/patched/prettytoml-unicode.patch +++ b/tasks/vendoring/patches/patched/prettytoml-unicode.patch @@ -11,8 +11,8 @@ index 8299195..2decd02 100644 import datetime import six @@ -81,10 +82,30 @@ def create_string_token(text, bare_string_allowed=False, multiline_strings_allow - - + + def _escape_single_line_quoted_string(text): - if six.PY2: - return text.encode('unicode-escape').encode('string-escape').replace('"', '\\"').replace("\\'", "'") @@ -42,8 +42,8 @@ index 8299195..2decd02 100644 + + flush() + return ''.join(res) - - + + def _create_multiline_string_token(text): diff --git a/pipenv/patched/prettytoml/tokens/toml2py.py b/pipenv/patched/prettytoml/tokens/toml2py.py index 2bf9c1c..5680443 100644 @@ -54,15 +54,6 @@ index 2bf9c1c..5680443 100644 import re import string import iso8601 -@@ -19,7 +20,7 @@ def deserialize(token): - - Raises DeserializationError when appropriate. - """ -- -+ - if token.type == TYPE_BOOLEAN: - return _to_boolean(token) - elif token.type == TYPE_INTEGER: @@ -39,42 +40,40 @@ def _unescape_str(text): """ Unescapes a string according the TOML spec. Raises BadEscapeCharacter when appropriate. @@ -136,6 +127,6 @@ index 2bf9c1c..5680443 100644 + tokens.append(escapes[text[i]]) + i += 1 + return ''.join(tokens) - - + + def _to_string(token): From 73243a8f012b82783752d91d0b0c6dc83a4236c9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 6 Nov 2018 21:36:50 -0500 Subject: [PATCH 097/218] Pass decoder to `toml.loads` Signed-off-by: Dan Ryan --- pipenv/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/project.py b/pipenv/project.py index da6d8560..322bdb9c 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -600,7 +600,7 @@ class Project(object): # Fallback to toml parser, for large files. except Exception: toml_decoder = toml.decoder.TomlDecoder() - return toml.loads(contents, encoder=toml_decoder) + return toml.loads(contents, decoder=toml_decoder) def _read_pyproject(self): pyproject = self.path_to("pyproject.toml") From bd1caee430e3c88b92af6492149ec49ea940e8e6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 6 Nov 2018 21:42:31 -0500 Subject: [PATCH 098/218] Update container patch for comment retention Signed-off-by: Dan Ryan --- pipenv/vendor/tomlkit/container.py | 7 ++++--- .../patches/vendor/tomlkit-update-items.patch | 13 ++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index 729083a0..a0014c6e 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -191,9 +191,10 @@ class Container(dict): old_data = self._body[idx][1] trivia = getattr(old_data, "trivia", None) - if trivia and idx > 0 and getattr(trivia, "comment", None): - self._body[idx - 1][1].comment(trivia.comment) - self._body[idx] = (None, Null()) + if trivia and getattr(trivia, "comment", None): + self._body[idx] = (None, trivia) + else: + self._body[idx] = (None, Null()) super(Container, self).__delitem__(key.key) diff --git a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch index 304b4375..2888ff6b 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch @@ -1,15 +1,18 @@ diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py -index 37014921..729083a0 100644 +index 37014921..a0014c6e 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py -@@ -189,6 +189,10 @@ class Container(dict): +@@ -189,7 +189,12 @@ class Container(dict): if idx is None: raise NonExistentKey(key) +- self._body[idx] = (None, Null()) + old_data = self._body[idx][1] + trivia = getattr(old_data, "trivia", None) -+ if trivia and idx > 0 and getattr(trivia, "comment", None): -+ self._body[idx - 1][1].comment(trivia.comment) - self._body[idx] = (None, Null()) ++ if trivia and getattr(trivia, "comment", None): ++ self._body[idx] = (None, trivia) ++ else: ++ self._body[idx] = (None, Null()) super(Container, self).__delitem__(key.key) + From 2f809c2fa3002b980eb1018903422202c954bf14 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 00:07:33 -0500 Subject: [PATCH 099/218] Fix tomlkit trivia patch Signed-off-by: Dan Ryan --- pipenv/vendor/tomlkit/container.py | 3 ++- .../patches/vendor/tomlkit-update-items.patch | 14 +++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index a0014c6e..a425cf79 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -9,6 +9,7 @@ from .items import Item from .items import Key from .items import Null from .items import Table +from .items import Trivia from .items import Whitespace from .items import item as _item @@ -192,7 +193,7 @@ class Container(dict): old_data = self._body[idx][1] trivia = getattr(old_data, "trivia", None) if trivia and getattr(trivia, "comment", None): - self._body[idx] = (None, trivia) + self._body[idx] = (None, Trivia(comment_ws="", comment=trivia.comment)) else: self._body[idx] = (None, Null()) diff --git a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch index 2888ff6b..10e4fcc1 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch @@ -1,8 +1,16 @@ diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py -index 37014921..a0014c6e 100644 +index 37014921..a425cf79 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py -@@ -189,7 +189,12 @@ class Container(dict): +@@ -9,6 +9,7 @@ from .items import Item + from .items import Key + from .items import Null + from .items import Table ++from .items import Trivia + from .items import Whitespace + from .items import item as _item + +@@ -189,7 +190,12 @@ class Container(dict): if idx is None: raise NonExistentKey(key) @@ -10,7 +18,7 @@ index 37014921..a0014c6e 100644 + old_data = self._body[idx][1] + trivia = getattr(old_data, "trivia", None) + if trivia and getattr(trivia, "comment", None): -+ self._body[idx] = (None, trivia) ++ self._body[idx] = (None, Trivia(comment_ws="", comment=trivia.comment)) + else: + self._body[idx] = (None, Null()) From 5908e804f183267ac8f5d1b54524a96dd6ef814c Mon Sep 17 00:00:00 2001 From: jxltom Date: Tue, 6 Nov 2018 13:13:25 +0800 Subject: [PATCH 100/218] Update skip lock option help info since it is used in several operations --- pipenv/cli/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index c3340bf8..1307eb6e 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -126,7 +126,7 @@ def skip_lock_option(f): state.installstate.skip_lock = value return value return option("--skip-lock", is_flag=True, default=False, expose_value=False, - help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", + help=u"Ignore locking mechanisms during operation—use the Pipfile, instead.", callback=callback, type=click.types.BOOL)(f) From e98c9a40500b4a4990753dd39f993a6f5ff6376a Mon Sep 17 00:00:00 2001 From: jxltom Date: Wed, 7 Nov 2018 13:19:04 +0800 Subject: [PATCH 101/218] Replace ignore instead of skip in skip-lock option's help info --- pipenv/cli/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 1307eb6e..3a361c5e 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -126,7 +126,7 @@ def skip_lock_option(f): state.installstate.skip_lock = value return value return option("--skip-lock", is_flag=True, default=False, expose_value=False, - help=u"Ignore locking mechanisms during operation—use the Pipfile, instead.", + help=u"Skip locking mechanisms and use the Pipfile instead during operation.", callback=callback, type=click.types.BOOL)(f) From 005c285dbac4145b672158e637c8ad36f6f5ddb5 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 00:28:05 -0500 Subject: [PATCH 102/218] Recursively build toml documents when parsing pipfile Signed-off-by: Dan Ryan --- pipenv/project.py | 113 +++++++++++++++++++++------------------------- 1 file changed, 51 insertions(+), 62 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 322bdb9c..9a08a161 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -577,30 +577,53 @@ class Project(object): """Clear pipfile cache (e.g., so we can mutate parsed pipfile)""" _pipfile_cache.clear() + @staticmethod + def dump_dict(dictionary, write_to, inline=False): + """ + Perform a nested recursive translation of a dictionary structure to a toml object. + + :param dictionary: A base dictionary to translate + :param write_to: The root node which will be mutated by the operation + :param inline: Whether to create inline tables for dictionaries, defaults to False + :return: A new toml hierarchical document + """ + + + def gen_table(inline=False): + if inline: + return tomlkit.inline_table() + return tomlkit.table() + + for key, value in dictionary.items(): + if isinstance(value, dict): + table = gen_table(inline=inline) + for sub_key, sub_value in value.items(): + if isinstance(sub_value, dict): + table[sub_key] = Project.dump_dict( + sub_value, gen_table(inline), inline=inline + ) + else: + table[sub_key] = sub_value + write_to[key] = table + else: + write_to[key] = Project.dump_dict(value, gen_table(inline), inline=inline) + else: + write_to[key] = value + return write_to + def _parse_pipfile(self, contents): # If any outline tables are present... - if ("[packages." in contents) or ("[dev-packages." in contents): + try: data = tomlkit.parse(contents) # Convert all outline tables to inline tables. for section in ("packages", "dev-packages"): - for package in data.get(section, {}): - # Convert things to inline tables — fancy :) - if hasattr(data[section][package], "keys"): - table = tomlkit.inline_table() - _data = data[section][package] - table.update(_data) - data[section][package] = table - # We lose comments here, but it's for the best.) + data = Project.dump_dict(data.get(section), tomlkit.table(), inline=True) + # We lose comments here, but it's for the best.) return data - - else: - try: - return tomlkit.parse(contents) - + except Exception: # Fallback to toml parser, for large files. - except Exception: - toml_decoder = toml.decoder.TomlDecoder() - return toml.loads(contents, decoder=toml_decoder) + toml_decoder = toml.decoder.TomlDecoder() + return toml.loads(contents, decoder=toml_decoder) def _read_pyproject(self): pyproject = self.path_to("pyproject.toml") @@ -967,59 +990,25 @@ class Project(object): name = self.get_package_name_in_pipfile(package_name, dev) key = "dev-packages" if dev else "packages" p = self.parsed_pipfile - try: - lines = [l for l in p.item(key).as_string().splitlines()] - except AttributeError: - lines = [l for l in p[key].serialized().splitlines()] - if not any(line.startswith("#") for line in lines) and name: + if name: del p[key][name] self.write_toml(p) - else: - p = self._pipfile - del p[key][name] - p.write() def remove_packages_from_pipfile(self, packages): - p = self._pipfile parsed = self.parsed_pipfile - packages = [pep423_name(pkg) for pkg in packages] - deleted_pkgs = [] - has_comments_as_lines = False + packages = set([pep423_name(pkg) for pkg in packages]) for section in ("dev-packages", "packages"): - pipfile_section = self.parsed_pipfile.get(section, {}) - try: - lines = [ - l for l in p.item(section).as_string().splitlines() - if section in parsed.keys() - ] - except AttributeError: - lines = [ - l for l in p[section].serialized().splitlines() - if section in parsed.keys() - ] - pipfile_packages = [ - pkg_name for pkg_name in pipfile_section.keys() - if pep423_name(pkg_name) in packages - ] + pipfile_section = parsed.get(section, {}) + pipfile_packages = set([ + pep423_name(pkg_name) for pkg_name in pipfile_section.keys() + ]) + to_remove = packages & pipfile_packages # The normal toml parser can't handle deleting packages with preceding newlines is_dev = section == "dev-packages" - if any(line.startswith("#") for line in lines): - has_comments_as_lines = True - for pkg in pipfile_packages: - pkg_name = self.get_package_name_in_pipfile(pkg, dev=is_dev) - deleted_pkgs.append(pkg) - del p.pipfile[section][pkg_name] - # However the alternative parser can't handle inline comment preservation - else: - for pkg in pipfile_packages: - pkg_name = self.get_package_name_in_pipfile(pkg, dev=is_dev) - deleted_pkgs.append(pkg) - del parsed[section][pkg_name] - if deleted_pkgs: - if has_comments_as_lines: - p.write() - else: - self.write_toml(parsed) + for pkg in to_remove: + pkg_name = self.get_package_name_in_pipfile(pkg, dev=is_dev) + del parsed[section][pkg_name] + self.write_toml(parsed) def add_package_to_pipfile(self, package, dev=False): from .vendor.requirementslib import Requirement From ca90be95d7124a05bab6ba8aa6bfa305ee9de1dd Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 00:45:47 -0500 Subject: [PATCH 103/218] Still not working... Signed-off-by: Dan Ryan --- pipenv/vendor/tomlkit/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index a425cf79..3934a05b 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -193,7 +193,7 @@ class Container(dict): old_data = self._body[idx][1] trivia = getattr(old_data, "trivia", None) if trivia and getattr(trivia, "comment", None): - self._body[idx] = (None, Trivia(comment_ws="", comment=trivia.comment)) + self._body[idx] = (None, trivia) else: self._body[idx] = (None, Null()) From 86a27869540a29fccbbe77e4b2def4647a68e58e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 7 Nov 2018 20:37:48 +0900 Subject: [PATCH 104/218] Quote command if it contains parentheses The command can be safely quoted in this case since non-quotable commands (cmd built-in) do not have parentheses. Note that this change only applies to the command, not arguments. Parentheses in arguments can be correctly interpreted without quotes, and the quotes, like spaces, will cause incorrect outputs for echo etc. --- news/3158.bugfix.rst | 1 + pipenv/cmdparse.py | 23 +++++++++++++++++------ tests/unit/test_cmdparse.py | 17 +++++++++++++++++ 3 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 news/3158.bugfix.rst diff --git a/news/3158.bugfix.rst b/news/3158.bugfix.rst new file mode 100644 index 00000000..661fb09e --- /dev/null +++ b/news/3158.bugfix.rst @@ -0,0 +1 @@ +Fix package installation when the virtual environment path contains parentheses. diff --git a/pipenv/cmdparse.py b/pipenv/cmdparse.py index 087a95b1..cec19273 100644 --- a/pipenv/cmdparse.py +++ b/pipenv/cmdparse.py @@ -1,3 +1,4 @@ +import itertools import re import shlex @@ -8,6 +9,12 @@ class ScriptEmptyError(ValueError): pass +def _quote_if_contains(value, pattern): + if next(re.finditer(pattern, value), None): + return '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', value)) + return value + + class Script(object): """Parse a script line (in Pipfile's [scripts] section). @@ -56,17 +63,21 @@ class Script(object): The result is then quoted into a pair of double quotes to be grouped. An argument is intentionally not quoted if it does not contain - whitespaces. This is done to be compatible with Windows built-in + foul characters. This is done to be compatible with Windows built-in commands that don't work well with quotes, e.g. everything with `echo`, and DOS-style (forward slash) switches. + Foul characters include: + + * Whitespaces. + * Parentheses in the command. (pypa/pipenv#3168) + The intended use of this function is to pre-process an argument list before passing it into ``subprocess.Popen(..., shell=True)``. See also: https://docs.python.org/3/library/subprocess.html#converting-argument-sequence """ - return " ".join( - arg if not next(re.finditer(r'\s', arg), None) - else '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', arg)) - for arg in self._parts - ) + return " ".join(itertools.chain( + [_quote_if_contains(self.command, r'[\s()]')], + (_quote_if_contains(arg, r'\s') for arg in self.args), + )) diff --git a/tests/unit/test_cmdparse.py b/tests/unit/test_cmdparse.py index 06012d07..1b329a53 100644 --- a/tests/unit/test_cmdparse.py +++ b/tests/unit/test_cmdparse.py @@ -47,3 +47,20 @@ def test_cmdify_complex(): '-c', """ "print(\'Double quote: \\\"\')" """.strip(), ]), script + + +@pytest.mark.run +@pytest.mark.script +def test_cmdify_quote_if_paren_in_command(): + """Ensure ONLY the command is quoted if it contains parentheses. + """ + script = Script.parse(' '.join([ + '"C:\\Python36(x86)\\python.exe"', + '-c', + "print(123)", + ])) + assert script.cmdify() == ' '.join([ + '"C:\\Python36(x86)\\python.exe"', + '-c', + "print(123)", + ]), script From d52134554e25a33d9519a912ec962224628f0d8a Mon Sep 17 00:00:00 2001 From: jxltom Date: Wed, 7 Nov 2018 22:42:22 +0800 Subject: [PATCH 105/218] Only output dev packages for pipenv lock -r -d --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 1c239e7f..758ef3fa 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -721,7 +721,7 @@ def do_install_dependencies( # Allow pip to resolve dependencies when in skip-lock mode. no_deps = not skip_lock failed_deps_list = [] - deps_list = list(lockfile.get_requirements(dev=dev, only=only)) + deps_list = list(lockfile.get_requirements(dev=dev, only=True)) if requirements: index_args = prepare_pip_source_args(project.sources) index_args = " ".join(index_args).replace(" -", "\n-") From d851b7ea2d66dff53f50e875e441faf15a16ede4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 10:39:31 -0500 Subject: [PATCH 106/218] Fix tomlkit implementation Signed-off-by: Dan Ryan --- pipenv/vendor/tomlkit/container.py | 4 ++-- .../patches/vendor/tomlkit-update-items.patch | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index 3934a05b..5ddd72e7 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -193,11 +193,11 @@ class Container(dict): old_data = self._body[idx][1] trivia = getattr(old_data, "trivia", None) if trivia and getattr(trivia, "comment", None): - self._body[idx] = (None, trivia) + self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) else: self._body[idx] = (None, Null()) + super(Container, self).__delitem__(key.key) - super(Container, self).__delitem__(key.key) return self diff --git a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch index 10e4fcc1..51f6006a 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py -index 37014921..a425cf79 100644 +index 37014921..5ddd72e7 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -9,6 +9,7 @@ from .items import Item @@ -10,7 +10,7 @@ index 37014921..a425cf79 100644 from .items import Whitespace from .items import item as _item -@@ -189,7 +190,12 @@ class Container(dict): +@@ -189,9 +190,14 @@ class Container(dict): if idx is None: raise NonExistentKey(key) @@ -18,9 +18,12 @@ index 37014921..a425cf79 100644 + old_data = self._body[idx][1] + trivia = getattr(old_data, "trivia", None) + if trivia and getattr(trivia, "comment", None): -+ self._body[idx] = (None, Trivia(comment_ws="", comment=trivia.comment)) ++ self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) + else: + self._body[idx] = (None, Null()) ++ super(Container, self).__delitem__(key.key) - super(Container, self).__delitem__(key.key) +- super(Container, self).__delitem__(key.key) + + return self From 4cb0838fda0e0bb850fdda82313f3bb64b7cd968 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 14:50:33 -0500 Subject: [PATCH 107/218] Fix tomlkit implementation Signed-off-by: Dan Ryan --- pipenv/project.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 9a08a161..26b4cf0c 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -617,10 +617,18 @@ class Project(object): data = tomlkit.parse(contents) # Convert all outline tables to inline tables. for section in ("packages", "dev-packages"): - data = Project.dump_dict(data.get(section), tomlkit.table(), inline=True) - # We lose comments here, but it's for the best.) + table_data = data.get(section, tomlkit.table()) + for package, value in table_data.items(): + if isinstance(value, dict): + table = tomlkit.inline_table() + table.update(value) + table_data[package] = table + else: + table_data[package] = value + data[section] = table_data return data except Exception: + # We lose comments here, but it's for the best.) # Fallback to toml parser, for large files. toml_decoder = toml.decoder.TomlDecoder() return toml.loads(contents, decoder=toml_decoder) From 2b195dd789250b57547fe58f168a3f39e57c98d4 Mon Sep 17 00:00:00 2001 From: devxpy Date: Thu, 8 Nov 2018 06:00:09 +0530 Subject: [PATCH 108/218] fix errors from merge --- pipenv/cli/options.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index fa20fc94..5c4a991f 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -126,7 +126,6 @@ def skip_lock_option(f): state.installstate.skip_lock = value return value return option("--skip-lock", is_flag=True, default=False, expose_value=False, - help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", help=u"Skip locking mechanisms and use the Pipfile instead during operation.", envvar="PIPENV_SKIP_LOCK", callback=callback, type=click.types.BOOL)(f) From e818ca5af63d7480f51385b4dd3c28dab197b643 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 19:26:38 -0500 Subject: [PATCH 109/218] Fix double completion message on locking - Pass `pre` correctly when using `pipenv install --pre` - Ensures we always make inline tables when writing to pipfile - Fixes #3183 - Fixes #3185 Signed-off-by: Dan Ryan --- news/3183.bugfix.rst | 1 + news/3185.bugfix.rst | 1 + pipenv/cli/options.py | 2 +- pipenv/core.py | 34 ++++++---------- pipenv/project.py | 15 ++++--- pipenv/utils.py | 95 ++++++++++++++++++++++++++++--------------- 6 files changed, 87 insertions(+), 61 deletions(-) create mode 100644 news/3183.bugfix.rst create mode 100644 news/3185.bugfix.rst diff --git a/news/3183.bugfix.rst b/news/3183.bugfix.rst new file mode 100644 index 00000000..19e1d54a --- /dev/null +++ b/news/3183.bugfix.rst @@ -0,0 +1 @@ +Fixed new spinner success message to write only one success message during resolution. diff --git a/news/3185.bugfix.rst b/news/3185.bugfix.rst new file mode 100644 index 00000000..b6ffee2b --- /dev/null +++ b/news/3185.bugfix.rst @@ -0,0 +1 @@ +Pipenv will now correctly respect the ``--pre`` option when used with ``pipenv install``. diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 3a361c5e..208c0c66 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -338,6 +338,7 @@ def common_options(f): def install_base_options(f): f = common_options(f) f = dev_option(f) + f = pre_option(f) f = keep_outdated_option(f) return f @@ -353,7 +354,6 @@ def uninstall_options(f): def lock_options(f): f = install_base_options(f) f = requirements_flag(f) - f = pre_option(f) return f diff --git a/pipenv/core.py b/pipenv/core.py index 758ef3fa..4a292139 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1028,10 +1028,22 @@ def do_lock( deps = convert_deps_to_pip( settings["packages"], project, r=False, include_index=True ) - results = venv_resolve_deps( + # Add refs for VCS installs. + # TODO: be smarter about this. + vcs_reqs, vcs_lockfile = get_vcs_deps( + project, + which=which, + clear=clear, + pre=pre, + allow_global=system, + dev=settings["dev"], + ) + vcs_lines = [req.as_line() for req in vcs_reqs if req.editable] + results, vcs_results = venv_resolve_deps( deps, which=which, project=project, + vcs_deps=vcs_lines, clear=clear, pre=pre, allow_global=system, @@ -1045,26 +1057,6 @@ def do_lock( dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry ) lockfile[settings["lockfile_key"]].update(dep_lockfile) - # Add refs for VCS installs. - # TODO: be smarter about this. - vcs_reqs, vcs_lockfile = get_vcs_deps( - project, - which=which, - clear=clear, - pre=pre, - allow_global=system, - dev=settings["dev"], - ) - vcs_lines = [req.as_line() for req in vcs_reqs if req.editable] - vcs_results = venv_resolve_deps( - vcs_lines, - which=which, - project=project, - clear=clear, - pre=pre, - allow_global=system, - pypi_mirror=pypi_mirror, - ) for dep in vcs_results: normalized = pep423_name(dep["name"]) if not hasattr(dep, "keys") or not hasattr(dep["name"], "keys"): diff --git a/pipenv/project.py b/pipenv/project.py index 26b4cf0c..98824d94 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -617,15 +617,14 @@ class Project(object): data = tomlkit.parse(contents) # Convert all outline tables to inline tables. for section in ("packages", "dev-packages"): - table_data = data.get(section, tomlkit.table()) + table_data = data.get(section, {}) for package, value in table_data.items(): if isinstance(value, dict): - table = tomlkit.inline_table() - table.update(value) - table_data[package] = table + package_table = tomlkit.inline_table() + package_table.update(value) + data[section][package] = package_table else: - table_data[package] = value - data[section] = table_data + data[section][package] = value return data except Exception: # We lose comments here, but it's for the best.) @@ -1036,6 +1035,10 @@ class Project(object): # Skip for wildcard version return # Add the package to the group. + if isinstance(converted, dict): + package_table = tomlkit.inline_table() + package_table.update(converted) + converted = package_table p[key][name or package.normalized_name] = converted # Write Pipfile. self.write_toml(p) diff --git a/pipenv/utils.py b/pipenv/utils.py index c9feeafd..26268ab2 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -468,6 +468,37 @@ def create_spinner(text, nospin=None, spinner_name=None): yield sp +def resolve(cmd, sp): + from .vendor import delegator + from .cmdparse import Script + from .vendor.pexpect.exceptions import EOF, TIMEOUT + from .vendor.vistir.compat import to_native_string + EOF.__module__ = "pexpect.exceptions" + from ._compat import decode_output + c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) + _out = decode_output("") + result = None + out = to_native_string("") + while True: + try: + result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT) + except (EOF, TIMEOUT): + pass + if result is None: + break + _out = c.subprocess.before + if _out is not None: + _out = decode_output("{0}".format(_out)) + out += _out + sp.text = to_native_string("{0}".format(_out[:100])) + if environments.is_verbose(): + if _out is not None: + sp._hide_cursor() + sp.write(_out.rstrip()) + sp._show_cursor() + return c + + def venv_resolve_deps( deps, which, @@ -476,19 +507,16 @@ def venv_resolve_deps( clear=False, allow_global=False, pypi_mirror=None, + vcs_deps=None, ): from .vendor.vistir.misc import fs_str - from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError + from .vendor.vistir.compat import Path, JSONDecodeError from .vendor.vistir.path import create_tracked_tempdir - from .cmdparse import Script - from .vendor.pexpect.exceptions import EOF, TIMEOUT - from .vendor import delegator from . import resolver - from ._compat import decode_output import json if not deps: - return [] + return [], [] req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") cmd = [ @@ -509,29 +537,8 @@ def venv_resolve_deps( os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") - out = to_native_string("") - EOF.__module__ = "pexpect.exceptions" with create_spinner(text=fs_str("Locking...")) as sp: - c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) - _out = decode_output("") - result = None - while True: - try: - result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT) - except (EOF, TIMEOUT): - pass - if result is None: - break - _out = c.subprocess.before - if _out is not None: - _out = decode_output("{0}".format(_out)) - out += _out - sp.text = to_native_string("{0}".format(_out[:100])) - if environments.is_verbose(): - if _out is not None: - sp._hide_cursor() - sp.write(_out.rstrip()) - sp._show_cursor() + c = resolve(cmd, sp) c.block() if c.return_code != 0: sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( @@ -540,17 +547,39 @@ def venv_resolve_deps( click_echo(c.out.strip(), err=True) click_echo(c.err.strip(), err=True) sys.exit(c.return_code) + results = c.out + if vcs_deps: + with temp_environ(): + os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) + vcs_c = resolve(cmd, sp) + c.block() + if c.return_code != 0: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Locking Failed!" + )) + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) + sys.exit(c.return_code) + vcs_results = vcs_c.out + vcs_err = vcs_c.err else: - sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + vcs_results = "" + vcs_err = "" + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + outputs = [results, vcs_results] if environments.is_verbose(): - click_echo(c.out.split("RESULTS:")[0], err=True) + for output in outputs: + click_echo(output.split("RESULTS:")[0], err=True) try: - return json.loads(c.out.split("RESULTS:")[1].strip()) + results = json.loads(results.split("RESULTS:")[1].strip()) + vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) except (IndexError, JSONDecodeError): - click_echo(c.out.strip(), err=True) - click_echo(c.err.strip(), err=True) + for out, err in [(c.out, c.err), (vcs_results, vcs_err)]: + click_echo(out.strip(), err=True) + click_echo(err.strip(), err=True) raise RuntimeError("There was a problem with locking.") + return results, vcs_results def resolve_deps( From b0e5fe12f95e17341563de75ae4bb603aaeb2702 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 20:20:58 -0500 Subject: [PATCH 110/218] Fix resolver Signed-off-by: Dan Ryan --- pipenv/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 26268ab2..73f15653 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -572,7 +572,10 @@ def venv_resolve_deps( click_echo(output.split("RESULTS:")[0], err=True) try: results = json.loads(results.split("RESULTS:")[1].strip()) - vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) + if vcs_results: + vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) + else: + vcs_results = [] except (IndexError, JSONDecodeError): for out, err in [(c.out, c.err), (vcs_results, vcs_err)]: From 3eba19d359ee5144f0d21c643b8617fe3fafbe92 Mon Sep 17 00:00:00 2001 From: frostming Date: Thu, 8 Nov 2018 10:22:26 +0800 Subject: [PATCH 111/218] Improve toml parsing --- pipenv/project.py | 70 ++++++++++--------------------- tests/integration/test_project.py | 18 ++++++++ tests/unit/test_vendor.py | 6 --- 3 files changed, 40 insertions(+), 54 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 26b4cf0c..4b90f493 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -8,7 +8,6 @@ import glob import base64 import fnmatch import hashlib -import contoml from first import first from cached_property import cached_property import operator @@ -578,60 +577,31 @@ class Project(object): _pipfile_cache.clear() @staticmethod - def dump_dict(dictionary, write_to, inline=False): - """ - Perform a nested recursive translation of a dictionary structure to a toml object. - - :param dictionary: A base dictionary to translate - :param write_to: The root node which will be mutated by the operation - :param inline: Whether to create inline tables for dictionaries, defaults to False - :return: A new toml hierarchical document - """ - - - def gen_table(inline=False): - if inline: - return tomlkit.inline_table() - return tomlkit.table() - - for key, value in dictionary.items(): - if isinstance(value, dict): - table = gen_table(inline=inline) - for sub_key, sub_value in value.items(): - if isinstance(sub_value, dict): - table[sub_key] = Project.dump_dict( - sub_value, gen_table(inline), inline=inline - ) - else: - table[sub_key] = sub_value - write_to[key] = table - else: - write_to[key] = Project.dump_dict(value, gen_table(inline), inline=inline) + def convert_outline_table(parsed): + """Converts all outline to inline tables""" + if hasattr(parsed, "_body"): # Duck-type that implies tomlkit.api.Container. + empty_inline_table = tomlkit.inline_table else: - write_to[key] = value - return write_to + empty_inline_table = toml.TomlDecoder().get_empty_inline_table + for section in ("packages", "dev-packages"): + table_data = parsed.get(section, {}) + for package, value in table_data.items(): + if hasattr(value, "keys"): + table = empty_inline_table() + table.update(value) + table_data[package] = table + return parsed def _parse_pipfile(self, contents): - # If any outline tables are present... try: data = tomlkit.parse(contents) - # Convert all outline tables to inline tables. - for section in ("packages", "dev-packages"): - table_data = data.get(section, tomlkit.table()) - for package, value in table_data.items(): - if isinstance(value, dict): - table = tomlkit.inline_table() - table.update(value) - table_data[package] = table - else: - table_data[package] = value - data[section] = table_data - return data except Exception: # We lose comments here, but it's for the best.) # Fallback to toml parser, for large files. - toml_decoder = toml.decoder.TomlDecoder() - return toml.loads(contents, decoder=toml_decoder) + data = toml.loads(contents) + if "[packages." in contents or "[dev-packages." in contents: + data = self.convert_outline_table(data) + return data def _read_pyproject(self): pyproject = self.path_to("pyproject.toml") @@ -886,7 +856,11 @@ class Project(object): if path is None: path = self.pipfile_location try: - formatted_data = tomlkit.dumps(data).rstrip() + if hasattr(data, "_body"): + formatted_data = tomlkit.dumps(data).rstrip() + else: + encoder = toml.encoder.TomlPreserveInlineDictEncoder() + formatted_data = toml.dumps(data, encoder=encoder) except Exception: document = tomlkit.document() for section in ("packages", "dev-packages"): diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 5deccc84..1e00bbb7 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -143,3 +143,21 @@ six = {{version = "*", index = "pypi"}} f.write(contents) c = p.pipenv('install') assert c.return_code == 0 + + +@pytest.mark.install +@pytest.mark.project +def test_rewrite_outline_table(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[packages.requests] +version = "*" + """.strip() + f.write(contents) + c = p.pipenv('install click') + assert c.return_code == 0 + with open(p.pipfile_path) as f: + contents = f.read() + assert "[packages.requests]" not in contents + assert 'requests = { version = "*" }' in contents diff --git a/tests/unit/test_vendor.py b/tests/unit/test_vendor.py index 1894b6ff..704b37fb 100644 --- a/tests/unit/test_vendor.py +++ b/tests/unit/test_vendor.py @@ -106,9 +106,3 @@ class TestPipfileParser: def test_token_date(dt, content): token = create_primitive_token(dt) assert token == tokens.Token(tokens.TYPE_DATE, content) - - -def test_dump_nonascii_string(): - content = 'name = "Stažené"\n' - toml_content = contoml.dumps(contoml.loads(content)) - assert toml_content == content From 59961d6bbbc7ad6ce9792f792fe2d913aa41322e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 21:54:05 -0500 Subject: [PATCH 112/218] Move VCS resolution to venv and inside spinner Signed-off-by: Dan Ryan --- pipenv/core.py | 14 ++----------- pipenv/utils.py | 54 ++++++++++++++++++++++++++++--------------------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 4a292139..5316733c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1028,27 +1028,17 @@ def do_lock( deps = convert_deps_to_pip( settings["packages"], project, r=False, include_index=True ) - # Add refs for VCS installs. - # TODO: be smarter about this. - vcs_reqs, vcs_lockfile = get_vcs_deps( - project, - which=which, - clear=clear, - pre=pre, - allow_global=system, - dev=settings["dev"], - ) - vcs_lines = [req.as_line() for req in vcs_reqs if req.editable] results, vcs_results = venv_resolve_deps( deps, which=which, project=project, - vcs_deps=vcs_lines, + dev=settings["dev"], clear=clear, pre=pre, allow_global=system, pypi_mirror=pypi_mirror, ) + vcs_results, vcs_lockfile = vcs_results # Add dependencies to lockfile. for dep in results: is_top_level = dep["name"] in settings["packages"] diff --git a/pipenv/utils.py b/pipenv/utils.py index 73f15653..444333de 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -496,6 +496,14 @@ def resolve(cmd, sp): sp._hide_cursor() sp.write(_out.rstrip()) sp._show_cursor() + c.block() + if c.return_code != 0: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Locking Failed!" + )) + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) + sys.exit(c.return_code) return c @@ -507,7 +515,7 @@ def venv_resolve_deps( clear=False, allow_global=False, pypi_mirror=None, - vcs_deps=None, + dev=False, ): from .vendor.vistir.misc import fs_str from .vendor.vistir.compat import Path, JSONDecodeError @@ -515,10 +523,28 @@ def venv_resolve_deps( from . import resolver import json + vcs_deps = [] + vcs_lockfile = {} + results = [] if not deps: - return [], [] + return results, (vcs_deps, vcs_lockfile) req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") + vcs_section = "vcs_dev_packages" if dev else "vcs_packages" + if getattr(project, vcs_section, []): + with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: + vcs_reqs, vcs_lockfile = get_vcs_deps( + project, + which=which, + clear=clear, + pre=pre, + allow_global=allow_global, + dev=dev, + ) + vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] + sp.write(environments.PIPENV_SPINNER_OK_TEXT.format( + "Successfully pinned VCS Packages!" + )) cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() @@ -539,32 +565,14 @@ def venv_resolve_deps( os.environ["PIP_NO_INPUT"] = fs_str("1") with create_spinner(text=fs_str("Locking...")) as sp: c = resolve(cmd, sp) - c.block() - if c.return_code != 0: - sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( - "Locking Failed!" - )) - click_echo(c.out.strip(), err=True) - click_echo(c.err.strip(), err=True) - sys.exit(c.return_code) results = c.out if vcs_deps: with temp_environ(): os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) vcs_c = resolve(cmd, sp) - c.block() - if c.return_code != 0: - sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( - "Locking Failed!" - )) - click_echo(c.out.strip(), err=True) - click_echo(c.err.strip(), err=True) - sys.exit(c.return_code) - vcs_results = vcs_c.out - vcs_err = vcs_c.err + vcs_results, vcs_err = vcs_c.out, vcs_c.err else: - vcs_results = "" - vcs_err = "" + vcs_results, vcs_err = "", "" sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) outputs = [results, vcs_results] if environments.is_verbose(): @@ -582,7 +590,7 @@ def venv_resolve_deps( click_echo(out.strip(), err=True) click_echo(err.strip(), err=True) raise RuntimeError("There was a problem with locking.") - return results, vcs_results + return results, (vcs_results, vcs_lockfile) def resolve_deps( From e147dc3ade4040e2b59c170c752f080839845264 Mon Sep 17 00:00:00 2001 From: frostming Date: Thu, 8 Nov 2018 10:57:20 +0800 Subject: [PATCH 113/218] Drops prettytoml/contoml form vendors --- news/3191.vendor.rst | 1 + pipenv/patched/contoml/LICENSE | 22 - pipenv/patched/contoml/__init__.py | 48 --- pipenv/patched/contoml/_version.py | 1 - pipenv/patched/contoml/file/__init__.py | 3 - pipenv/patched/contoml/file/array.py | 40 -- pipenv/patched/contoml/file/cascadedict.py | 56 --- pipenv/patched/contoml/file/file.py | 293 -------------- pipenv/patched/contoml/file/freshtable.py | 45 --- pipenv/patched/contoml/file/peekableit.py | 30 -- pipenv/patched/contoml/file/raw.py | 16 - pipenv/patched/contoml/file/structurer.py | 116 ------ .../patched/contoml/file/test_cascadedict.py | 25 -- pipenv/patched/contoml/file/test_entries.py | 20 - .../patched/contoml/file/test_peekableit.py | 12 - .../patched/contoml/file/test_structurer.py | 41 -- pipenv/patched/contoml/file/toplevels.py | 142 ------- pipenv/patched/patched.txt | 1 - pipenv/patched/prettytoml/LICENSE | 22 - pipenv/patched/prettytoml/__init__.py | 25 -- pipenv/patched/prettytoml/_version.py | 1 - .../patched/prettytoml/elements/__init__.py | 13 - .../prettytoml/elements/abstracttable.py | 92 ----- pipenv/patched/prettytoml/elements/array.py | 136 ------- pipenv/patched/prettytoml/elements/atomic.py | 52 --- pipenv/patched/prettytoml/elements/common.py | 101 ----- pipenv/patched/prettytoml/elements/errors.py | 13 - pipenv/patched/prettytoml/elements/factory.py | 152 ------- .../prettytoml/elements/inlinetable.py | 78 ---- .../patched/prettytoml/elements/metadata.py | 80 ---- pipenv/patched/prettytoml/elements/table.py | 122 ------ .../prettytoml/elements/tableheader.py | 95 ----- .../patched/prettytoml/elements/test_array.py | 67 ---- .../prettytoml/elements/test_atomic.py | 9 - .../prettytoml/elements/test_common.py | 89 ----- .../prettytoml/elements/test_factory.py | 22 - .../prettytoml/elements/test_inlinetable.py | 52 --- .../prettytoml/elements/test_metadata.py | 25 -- .../patched/prettytoml/elements/test_table.py | 59 --- .../prettytoml/elements/test_tableheader.py | 12 - .../prettytoml/elements/test_traversal.py | 18 - .../prettytoml/elements/traversal/__init__.py | 175 -------- .../elements/traversal/predicates.py | 48 --- pipenv/patched/prettytoml/errors.py | 32 -- pipenv/patched/prettytoml/lexer/__init__.py | 123 ------ pipenv/patched/prettytoml/lexer/test_lexer.py | 153 ------- pipenv/patched/prettytoml/parser/__init__.py | 34 -- .../prettytoml/parser/elementsanitizer.py | 58 --- pipenv/patched/prettytoml/parser/errors.py | 17 - pipenv/patched/prettytoml/parser/parser.py | 376 ------------------ pipenv/patched/prettytoml/parser/recdesc.py | 114 ------ .../patched/prettytoml/parser/test_parser.py | 156 -------- .../patched/prettytoml/parser/tokenstream.py | 39 -- .../patched/prettytoml/prettifier/__init__.py | 39 -- .../prettytoml/prettifier/commentspace.py | 35 -- .../patched/prettytoml/prettifier/common.py | 54 --- .../prettifier/deindentanonymoustable.py | 43 -- .../prettytoml/prettifier/linelength.py | 62 --- .../prettytoml/prettifier/tableassignment.py | 40 -- .../prettytoml/prettifier/tableentrysort.py | 38 -- .../prettytoml/prettifier/tableindent.py | 49 --- .../patched/prettytoml/prettifier/tablesep.py | 31 -- .../prettifier/test_commentspace.py | 28 -- .../prettifier/test_deindentanonymoustable.py | 22 - .../prettytoml/prettifier/test_linelength.py | 39 -- .../prettifier/test_tableassignment.py | 29 -- .../prettifier/test_tableentrysort.py | 45 --- .../prettytoml/prettifier/test_tableindent.py | 25 -- .../prettytoml/prettifier/test_tablesep.py | 34 -- pipenv/patched/prettytoml/test_prettifier.py | 12 - pipenv/patched/prettytoml/test_util.py | 22 - pipenv/patched/prettytoml/tokens/__init__.py | 136 ------- pipenv/patched/prettytoml/tokens/errors.py | 13 - pipenv/patched/prettytoml/tokens/py2toml.py | 154 ------- .../patched/prettytoml/tokens/test_py2toml.py | 69 ---- .../patched/prettytoml/tokens/test_toml2py.py | 86 ---- pipenv/patched/prettytoml/tokens/toml2py.py | 130 ------ pipenv/patched/prettytoml/util.py | 141 ------- pipenv/project.py | 8 +- tasks/vendoring/patches/patched/contoml.patch | 28 -- .../patched/prettytoml-newlinefix.patch | 13 - .../patches/patched/prettytoml-python37.patch | 32 -- .../patched/prettytoml-table-iter.patch | 29 -- .../patches/patched/prettytoml-unicode.patch | 132 ------ .../patches/patched/prettytoml.patch | 78 ---- tests/integration/test_project.py | 2 +- tests/unit/test_vendor.py | 70 +--- 87 files changed, 5 insertions(+), 5335 deletions(-) create mode 100644 news/3191.vendor.rst delete mode 100644 pipenv/patched/contoml/LICENSE delete mode 100644 pipenv/patched/contoml/__init__.py delete mode 100644 pipenv/patched/contoml/_version.py delete mode 100644 pipenv/patched/contoml/file/__init__.py delete mode 100644 pipenv/patched/contoml/file/array.py delete mode 100644 pipenv/patched/contoml/file/cascadedict.py delete mode 100644 pipenv/patched/contoml/file/file.py delete mode 100644 pipenv/patched/contoml/file/freshtable.py delete mode 100644 pipenv/patched/contoml/file/peekableit.py delete mode 100644 pipenv/patched/contoml/file/raw.py delete mode 100644 pipenv/patched/contoml/file/structurer.py delete mode 100644 pipenv/patched/contoml/file/test_cascadedict.py delete mode 100644 pipenv/patched/contoml/file/test_entries.py delete mode 100644 pipenv/patched/contoml/file/test_peekableit.py delete mode 100644 pipenv/patched/contoml/file/test_structurer.py delete mode 100644 pipenv/patched/contoml/file/toplevels.py delete mode 100644 pipenv/patched/prettytoml/LICENSE delete mode 100644 pipenv/patched/prettytoml/__init__.py delete mode 100644 pipenv/patched/prettytoml/_version.py delete mode 100644 pipenv/patched/prettytoml/elements/__init__.py delete mode 100644 pipenv/patched/prettytoml/elements/abstracttable.py delete mode 100644 pipenv/patched/prettytoml/elements/array.py delete mode 100644 pipenv/patched/prettytoml/elements/atomic.py delete mode 100644 pipenv/patched/prettytoml/elements/common.py delete mode 100644 pipenv/patched/prettytoml/elements/errors.py delete mode 100644 pipenv/patched/prettytoml/elements/factory.py delete mode 100644 pipenv/patched/prettytoml/elements/inlinetable.py delete mode 100644 pipenv/patched/prettytoml/elements/metadata.py delete mode 100644 pipenv/patched/prettytoml/elements/table.py delete mode 100644 pipenv/patched/prettytoml/elements/tableheader.py delete mode 100644 pipenv/patched/prettytoml/elements/test_array.py delete mode 100644 pipenv/patched/prettytoml/elements/test_atomic.py delete mode 100644 pipenv/patched/prettytoml/elements/test_common.py delete mode 100644 pipenv/patched/prettytoml/elements/test_factory.py delete mode 100644 pipenv/patched/prettytoml/elements/test_inlinetable.py delete mode 100644 pipenv/patched/prettytoml/elements/test_metadata.py delete mode 100644 pipenv/patched/prettytoml/elements/test_table.py delete mode 100644 pipenv/patched/prettytoml/elements/test_tableheader.py delete mode 100644 pipenv/patched/prettytoml/elements/test_traversal.py delete mode 100644 pipenv/patched/prettytoml/elements/traversal/__init__.py delete mode 100644 pipenv/patched/prettytoml/elements/traversal/predicates.py delete mode 100644 pipenv/patched/prettytoml/errors.py delete mode 100644 pipenv/patched/prettytoml/lexer/__init__.py delete mode 100644 pipenv/patched/prettytoml/lexer/test_lexer.py delete mode 100644 pipenv/patched/prettytoml/parser/__init__.py delete mode 100644 pipenv/patched/prettytoml/parser/elementsanitizer.py delete mode 100644 pipenv/patched/prettytoml/parser/errors.py delete mode 100644 pipenv/patched/prettytoml/parser/parser.py delete mode 100644 pipenv/patched/prettytoml/parser/recdesc.py delete mode 100644 pipenv/patched/prettytoml/parser/test_parser.py delete mode 100644 pipenv/patched/prettytoml/parser/tokenstream.py delete mode 100644 pipenv/patched/prettytoml/prettifier/__init__.py delete mode 100644 pipenv/patched/prettytoml/prettifier/commentspace.py delete mode 100644 pipenv/patched/prettytoml/prettifier/common.py delete mode 100644 pipenv/patched/prettytoml/prettifier/deindentanonymoustable.py delete mode 100644 pipenv/patched/prettytoml/prettifier/linelength.py delete mode 100644 pipenv/patched/prettytoml/prettifier/tableassignment.py delete mode 100644 pipenv/patched/prettytoml/prettifier/tableentrysort.py delete mode 100644 pipenv/patched/prettytoml/prettifier/tableindent.py delete mode 100644 pipenv/patched/prettytoml/prettifier/tablesep.py delete mode 100644 pipenv/patched/prettytoml/prettifier/test_commentspace.py delete mode 100644 pipenv/patched/prettytoml/prettifier/test_deindentanonymoustable.py delete mode 100644 pipenv/patched/prettytoml/prettifier/test_linelength.py delete mode 100644 pipenv/patched/prettytoml/prettifier/test_tableassignment.py delete mode 100644 pipenv/patched/prettytoml/prettifier/test_tableentrysort.py delete mode 100644 pipenv/patched/prettytoml/prettifier/test_tableindent.py delete mode 100644 pipenv/patched/prettytoml/prettifier/test_tablesep.py delete mode 100644 pipenv/patched/prettytoml/test_prettifier.py delete mode 100644 pipenv/patched/prettytoml/test_util.py delete mode 100644 pipenv/patched/prettytoml/tokens/__init__.py delete mode 100644 pipenv/patched/prettytoml/tokens/errors.py delete mode 100644 pipenv/patched/prettytoml/tokens/py2toml.py delete mode 100644 pipenv/patched/prettytoml/tokens/test_py2toml.py delete mode 100644 pipenv/patched/prettytoml/tokens/test_toml2py.py delete mode 100644 pipenv/patched/prettytoml/tokens/toml2py.py delete mode 100644 pipenv/patched/prettytoml/util.py delete mode 100644 tasks/vendoring/patches/patched/contoml.patch delete mode 100644 tasks/vendoring/patches/patched/prettytoml-newlinefix.patch delete mode 100644 tasks/vendoring/patches/patched/prettytoml-python37.patch delete mode 100644 tasks/vendoring/patches/patched/prettytoml-table-iter.patch delete mode 100644 tasks/vendoring/patches/patched/prettytoml-unicode.patch delete mode 100644 tasks/vendoring/patches/patched/prettytoml.patch diff --git a/news/3191.vendor.rst b/news/3191.vendor.rst new file mode 100644 index 00000000..3806b68f --- /dev/null +++ b/news/3191.vendor.rst @@ -0,0 +1 @@ +Switch to ``tomlkit`` for parsing and writing. Drop ``prettytoml`` and ``contoml`` from vendors. diff --git a/pipenv/patched/contoml/LICENSE b/pipenv/patched/contoml/LICENSE deleted file mode 100644 index 116fa4e5..00000000 --- a/pipenv/patched/contoml/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Jumpscale - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/pipenv/patched/contoml/__init__.py b/pipenv/patched/contoml/__init__.py deleted file mode 100644 index 9dba5e20..00000000 --- a/pipenv/patched/contoml/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -from ._version import VERSION - -__version__ = VERSION - - -def loads(text): - """ - Parses TOML text into a dict-like object and returns it. - """ - from prettytoml.parser import parse_tokens - from prettytoml.lexer import tokenize as lexer - from .file import TOMLFile - - tokens = tuple(lexer(text, is_top_level=True)) - elements = parse_tokens(tokens) - return TOMLFile(elements) - - -def load(file_path): - """ - Parses a TOML file into a dict-like object and returns it. - """ - return loads(open(file_path).read()) - - -def dumps(value): - """ - Dumps a data structure to TOML source code. - - The given value must be either a dict of dict values, a dict, or a TOML file constructed by this module. - """ - - from contoml.file.file import TOMLFile - - if not isinstance(value, TOMLFile): - raise RuntimeError("Can only dump a TOMLFile instance loaded by load() or loads()") - - return value.dumps() - - -def dump(obj, file_path, prettify=False): - """ - Dumps a data structure to the filesystem as TOML. - - The given value must be either a dict of dict values, a dict, or a TOML file constructed by this module. - """ - with open(file_path, 'w') as fp: - fp.write(dumps(obj)) diff --git a/pipenv/patched/contoml/_version.py b/pipenv/patched/contoml/_version.py deleted file mode 100644 index e0f15470..00000000 --- a/pipenv/patched/contoml/_version.py +++ /dev/null @@ -1 +0,0 @@ -VERSION = 'master' diff --git a/pipenv/patched/contoml/file/__init__.py b/pipenv/patched/contoml/file/__init__.py deleted file mode 100644 index 1aba5121..00000000 --- a/pipenv/patched/contoml/file/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ - - -from .file import TOMLFile diff --git a/pipenv/patched/contoml/file/array.py b/pipenv/patched/contoml/file/array.py deleted file mode 100644 index 40e5acb3..00000000 --- a/pipenv/patched/contoml/file/array.py +++ /dev/null @@ -1,40 +0,0 @@ -from prettytoml.elements.table import TableElement -from prettytoml.errors import InvalidValueError -from contoml.file.freshtable import FreshTable -from prettytoml import util - - -class ArrayOfTables(list): - - def __init__(self, toml_file, name, iterable=None): - if iterable: - list.__init__(self, iterable) - self._name = name - self._toml_file = toml_file - - def append(self, value): - if isinstance(value, dict): - table = FreshTable(parent=self, name=self._name, is_array=True) - table._append_to_parent() - index = len(self._toml_file[self._name]) - 1 - for key_seq, value in util.flatten_nested(value).items(): - # self._toml_file._setitem_with_key_seq((self._name, index) + key_seq, value) - self._toml_file._array_setitem_with_key_seq(self._name, index, key_seq, value) - # for k, v in value.items(): - # table[k] = v - else: - raise InvalidValueError('Can only append a dict to an array of tables') - - def __getitem__(self, item): - try: - return list.__getitem__(self, item) - except IndexError: - if item == len(self): - return FreshTable(parent=self, name=self._name, is_array=True) - else: - raise - - def append_fresh_table(self, fresh_table): - list.append(self, fresh_table) - if self._toml_file: - self._toml_file.append_fresh_table(fresh_table) diff --git a/pipenv/patched/contoml/file/cascadedict.py b/pipenv/patched/contoml/file/cascadedict.py deleted file mode 100644 index 4e97c060..00000000 --- a/pipenv/patched/contoml/file/cascadedict.py +++ /dev/null @@ -1,56 +0,0 @@ -import operator -from functools import reduce -from contoml.file import raw - - -class CascadeDict: - """ - A dict-like object made up of one or more other dict-like objects where querying for an item cascade-gets - it from all the internal dicts in order of their listing, and setting an item sets it on the first dict listed. - """ - - def __init__(self, *internal_dicts): - assert internal_dicts, 'internal_dicts cannot be empty' - self._internal_dicts = tuple(internal_dicts) - - def cascaded_with(self, one_more_dict): - """ - Returns another instance with one more dict cascaded at the end. - """ - return CascadeDict(self._internal_dicts, one_more_dict,) - - def __getitem__(self, item): - for d in self._internal_dicts: - try: - return d[item] - except KeyError: - pass - raise KeyError - - def __setitem__(self, key, value): - self._internal_dicts[0][key] = value - - def keys(self): - return set(reduce(operator.or_, (set(d.keys()) for d in self._internal_dicts))) - - def items(self): - all_items = reduce(operator.add, (list(d.items()) for d in reversed(self._internal_dicts))) - unique_items = {k: v for k, v in all_items}.items() - return tuple(unique_items) - - def __contains__(self, item): - for d in self._internal_dicts: - if item in d: - return True - return False - - @property - def neutralized(self): - return {k: raw.to_raw(v) for k, v in self.items()} - - @property - def primitive_value(self): - return self.neutralized - - def __repr__(self): - return repr(self.primitive_value) diff --git a/pipenv/patched/contoml/file/file.py b/pipenv/patched/contoml/file/file.py deleted file mode 100644 index 99ce1483..00000000 --- a/pipenv/patched/contoml/file/file.py +++ /dev/null @@ -1,293 +0,0 @@ -from prettytoml.errors import NoArrayFoundError, DuplicateKeysError, DuplicateTablesError -from contoml.file import structurer, toplevels, raw -from contoml.file.array import ArrayOfTables -from contoml.file.freshtable import FreshTable -import prettytoml.elements.factory as element_factory -import prettytoml.util as util - - -class TOMLFile: - """ - A TOMLFile object that tries its best to prserve formatting and order of mappings of the input source. - - Raises InvalidTOMLFileError on invalid input elements. - - Raises DuplicateKeysError, DuplicateTableError when appropriate. - """ - - def __init__(self, _elements): - self._elements = [] - self._navigable = {} - self.append_elements(_elements) - - def __getitem__(self, item): - try: - value = self._navigable[item] - if isinstance(value, (list, tuple)): - return ArrayOfTables(toml_file=self, name=item, iterable=value) - else: - return value - except KeyError: - return FreshTable(parent=self, name=item, is_array=False) - - def get(self, item, default=None): - """This was not here for who knows why.""" - - if item not in self: - return default - else: - return self.__getitem__(item) - - def __contains__(self, item): - return item in self.keys() - - def _setitem_with_key_seq(self, key_seq, value): - """ - Sets a the value in the TOML file located by the given key sequence. - - Example: - self._setitem(('key1', 'key2', 'key3'), 'text_value') - is equivalent to doing - self['key1']['key2']['key3'] = 'text_value' - """ - table = self - key_so_far = tuple() - for key in key_seq[:-1]: - key_so_far += (key,) - self._make_sure_table_exists(key_so_far) - table = table[key] - table[key_seq[-1]] = value - - def _array_setitem_with_key_seq(self, array_name, index, key_seq, value): - """ - Sets a the array value in the TOML file located by the given key sequence. - - Example: - self._array_setitem(array_name, index, ('key1', 'key2', 'key3'), 'text_value') - is equivalent to doing - self.array(array_name)[index]['key1']['key2']['key3'] = 'text_value' - """ - table = self.array(array_name)[index] - key_so_far = tuple() - for key in key_seq[:-1]: - key_so_far += (key,) - new_table = self._array_make_sure_table_exists(array_name, index, key_so_far) - if new_table is not None: - table = new_table - else: - table = table[key] - table[key_seq[-1]] = value - - def _make_sure_table_exists(self, name_seq): - """ - Makes sure the table with the full name comprising of name_seq exists. - """ - t = self - for key in name_seq[:-1]: - t = t[key] - name = name_seq[-1] - if name not in t: - self.append_elements([element_factory.create_table_header_element(name_seq), - element_factory.create_table({})]) - - def _array_make_sure_table_exists(self, array_name, index, name_seq): - """ - Makes sure the table with the full name comprising of name_seq exists. - """ - t = self[array_name][index] - for key in name_seq[:-1]: - t = t[key] - name = name_seq[-1] - if name not in t: - new_table = element_factory.create_table({}) - self.append_elements([element_factory.create_table_header_element((array_name,) + name_seq), new_table]) - return new_table - - def __delitem__(self, key): - table_element_index = self._elements.index(self._navigable[key]) - self._elements[table_element_index] = element_factory.create_table({}) - self._on_element_change() - - def __setitem__(self, key, value): - - # Setting an array-of-tables - if key and isinstance(value, (tuple, list)) and value and all(isinstance(v, dict) for v in value): - for table in value: - self.array(key).append(table) - - # Or setting a whole single table - elif isinstance(value, dict): - - if key and key in self: - del self[key] - - for key_seq, child_value in util.flatten_nested({key: value}).items(): - self._setitem_with_key_seq(key_seq, child_value) - - # if key in self._navigable: - # del self[key] - # index = self._elements.index(self._navigable[key]) - # self._elements = self._elements[:index] + [element_factory.create_table(value)] + self._elements[index+1:] - # else: - # if key: - # self._elements.append(element_factory.create_table_header_element(key)) - # self._elements.append(element_factory.create_table(value)) - - - # Or updating the anonymous section table - else: - # It's mea - self[''][key] = value - - self._on_element_change() - - def _detect_toplevels(self): - """ - Returns a sequence of TopLevel instances for the current state of this table. - """ - return tuple(e for e in toplevels.identify(self.elements) if isinstance(e, toplevels.Table)) - - def _update_table_fallbacks(self, table_toplevels): - """ - Updates the fallbacks on all the table elements to make relative table access possible. - - Raises DuplicateKeysError if appropriate. - """ - - if len(self.elements) <= 1: - return - - def parent_of(toplevel): - # Returns an TopLevel parent of the given entry, or None. - for parent_toplevel in table_toplevels: - if toplevel.name.sub_names[:-1] == parent_toplevel.name.sub_names: - return parent_toplevel - - for entry in table_toplevels: - if entry.name.is_qualified: - parent = parent_of(entry) - if parent: - child_name = entry.name.without_prefix(parent.name) - parent.table_element.set_fallback({child_name.sub_names[0]: entry.table_element}) - - def _recreate_navigable(self): - if self._elements: - self._navigable = structurer.structure(toplevels.identify(self._elements)) - - def array(self, name): - """ - Returns the array of tables with the given name. - """ - if name in self._navigable: - if isinstance(self._navigable[name], (list, tuple)): - return self[name] - else: - raise NoArrayFoundError - else: - return ArrayOfTables(toml_file=self, name=name) - - def _on_element_change(self): - self._recreate_navigable() - - table_toplevels = self._detect_toplevels() - self._update_table_fallbacks(table_toplevels) - - def append_elements(self, elements): - """ - Appends more elements to the contained internal elements. - """ - self._elements = self._elements + list(elements) - self._on_element_change() - - def prepend_elements(self, elements): - """ - Prepends more elements to the contained internal elements. - """ - self._elements = list(elements) + self._elements - self._on_element_change() - - def dumps(self): - """ - Returns the TOML file serialized back to str. - """ - return ''.join(element.serialized() for element in self._elements) - - def dump(self, file_path): - with open(file_path, mode='w') as fp: - fp.write(self.dumps()) - - def keys(self): - return set(self._navigable.keys()) | {''} - - def values(self): - return self._navigable.values() - - def items(self): - items = self._navigable.items() - - def has_anonymous_entry(): - return any(key == '' for (key, _) in items) - - if has_anonymous_entry(): - return items - else: - return list(items) + [('', self[''])] - - @property - def primitive(self): - """ - Returns a primitive object representation for this container (which is a dict). - - WARNING: The returned container does not contain any markup or formatting metadata. - """ - raw_container = raw.to_raw(self._navigable) - - # Collapsing the anonymous table onto the top-level container is present - if '' in raw_container: - raw_container.update(raw_container['']) - del raw_container[''] - - return raw_container - - def append_fresh_table(self, fresh_table): - """ - Gets called by FreshTable instances when they get written to. - """ - if fresh_table.name: - elements = [] - if fresh_table.is_array: - elements += [element_factory.create_array_of_tables_header_element(fresh_table.name)] - else: - elements += [element_factory.create_table_header_element(fresh_table.name)] - - elements += [fresh_table, element_factory.create_newline_element()] - self.append_elements(elements) - - else: - # It's an anonymous table - self.prepend_elements([fresh_table, element_factory.create_newline_element()]) - - @property - def elements(self): - return self._elements - - def __str__(self): - - is_empty = (not self['']) and (not tuple(k for k in self.keys() if k)) - - def key_name(key): - return '[ANONYMOUS]' if not key else key - - def pair(key, value): - return '%s = %s' % (key_name(key), str(value)) - - content_text = '' if is_empty else \ - '\n\t' + ',\n\t'.join(pair(k, v) for (k, v) in self.items() if v) + '\n' - - return "TOMLFile{%s}" % content_text - - def __repr__(self): - return str(self) - - - diff --git a/pipenv/patched/contoml/file/freshtable.py b/pipenv/patched/contoml/file/freshtable.py deleted file mode 100644 index f28e2f74..00000000 --- a/pipenv/patched/contoml/file/freshtable.py +++ /dev/null @@ -1,45 +0,0 @@ -from prettytoml.elements.table import TableElement - - -class FreshTable(TableElement): - """ - A fresh TableElement that appended itself to each of parents when it first gets written to at most once. - - parents is a sequence of objects providing an append_fresh_table(TableElement) method - """ - - def __init__(self, parent, name, is_array=False): - TableElement.__init__(self, sub_elements=[]) - - self._parent = parent - self._name = name - self._is_array = is_array - - # As long as this flag is false, setitem() operations will append the table header and this table - # to the toml_file's elements - self.__appended = False - - @property - def name(self): - return self._name - - @property - def is_array(self): - return self._is_array - - def _append_to_parent(self): - """ - Causes this ephemeral table to be persisted on the TOMLFile. - """ - - if self.__appended: - return - - if self._parent is not None: - self._parent.append_fresh_table(self) - - self.__appended = True - - def __setitem__(self, key, value): - TableElement.__setitem__(self, key, value) - self._append_to_parent() diff --git a/pipenv/patched/contoml/file/peekableit.py b/pipenv/patched/contoml/file/peekableit.py deleted file mode 100644 index b5658a71..00000000 --- a/pipenv/patched/contoml/file/peekableit.py +++ /dev/null @@ -1,30 +0,0 @@ -import itertools - - -class PeekableIterator: - - # Returned by peek() when the iterator is exhausted. Truthiness is False. - Nothing = tuple() - - def __init__(self, iter): - self._iter = iter - - def __next__(self): - return next(self._iter) - - def next(self): - return self.__next__() - - def __iter__(self): - return self - - def peek(self): - """ - Returns PeekableIterator.Nothing when the iterator is exhausted. - """ - try: - v = next(self._iter) - self._iter = itertools.chain((v,), self._iter) - return v - except StopIteration: - return PeekableIterator.Nothing diff --git a/pipenv/patched/contoml/file/raw.py b/pipenv/patched/contoml/file/raw.py deleted file mode 100644 index 8cffdb6e..00000000 --- a/pipenv/patched/contoml/file/raw.py +++ /dev/null @@ -1,16 +0,0 @@ -from prettytoml.elements.abstracttable import AbstractTable - - -def to_raw(x): - from contoml.file.cascadedict import CascadeDict - - if isinstance(x, AbstractTable): - return x.primitive_value - elif isinstance(x, CascadeDict): - return x.neutralized - elif isinstance(x, (list, tuple)): - return [to_raw(y) for y in x] - elif isinstance(x, dict): - return {k: to_raw(v) for (k, v) in x.items()} - else: - return x diff --git a/pipenv/patched/contoml/file/structurer.py b/pipenv/patched/contoml/file/structurer.py deleted file mode 100644 index 72d002cd..00000000 --- a/pipenv/patched/contoml/file/structurer.py +++ /dev/null @@ -1,116 +0,0 @@ -from contoml.file import toplevels -from contoml.file.cascadedict import CascadeDict - - -class NamedDict(dict): - """ - A dict that can use Name instances as keys. - """ - - def __init__(self, other_dict=None): - dict.__init__(self) - if other_dict: - for k, v in other_dict.items(): - self[k] = v - - def __setitem__(self, key, value): - """ - key can be an Name instance. - - When key is a path in the form of an Name instance, all the parents and grandparents of the value are - created along the way as instances of NamedDict. If the parent of the value exists, it is replaced with a - CascadeDict() that cascades the old parent value with a new NamedDict that contains the given child name - and value. - """ - if isinstance(key, toplevels.Name): - - if len(key.sub_names) == 1: - name = key.sub_names[0] - if name in self: - self[name] = CascadeDict(self[name], value) - else: - self[name] = value - - elif len(key.sub_names) > 1: - name = key.sub_names[0] - rest_of_key = key.drop(1) - if name in self: - named_dict = NamedDict() - named_dict[rest_of_key] = value - self[name] = CascadeDict(self[name], named_dict) - else: - self[name] = NamedDict() - self[name][rest_of_key] = value - else: - return dict.__setitem__(self, key, value) - - def __contains__(self, item): - try: - _ = self[item] - return True - except KeyError: - return False - - def append(self, key, value): - """ - Makes sure the value pointed to by key exists and is a list and appends the given value to it. - """ - if key in self: - self[key].append(value) - else: - self[key] = [value] - - def __getitem__(self, item): - - if isinstance(item, toplevels.Name): - d = self - for name in item.sub_names: - d = d[name] - return d - else: - return dict.__getitem__(self, item) - - -def structure(table_toplevels): - """ - Accepts an ordered sequence of TopLevel instances and returns a navigable object structure representation of the - TOML file. - """ - - table_toplevels = tuple(table_toplevels) - obj = NamedDict() - - last_array_of_tables = None # The Name of the last array-of-tables header - - for toplevel in table_toplevels: - - if isinstance(toplevel, toplevels.AnonymousTable): - obj[''] = toplevel.table_element - - elif isinstance(toplevel, toplevels.Table): - if last_array_of_tables and toplevel.name.is_prefixed_with(last_array_of_tables): - seq = obj[last_array_of_tables] - unprefixed_name = toplevel.name.without_prefix(last_array_of_tables) - - seq[-1] = CascadeDict(seq[-1], NamedDict({unprefixed_name: toplevel.table_element})) - else: - obj[toplevel.name] = toplevel.table_element - else: # It's an ArrayOfTables - - if last_array_of_tables and toplevel.name != last_array_of_tables and \ - toplevel.name.is_prefixed_with(last_array_of_tables): - - seq = obj[last_array_of_tables] - unprefixed_name = toplevel.name.without_prefix(last_array_of_tables) - - if unprefixed_name in seq[-1]: - seq[-1][unprefixed_name].append(toplevel.table_element) - else: - cascaded_with = NamedDict({unprefixed_name: [toplevel.table_element]}) - seq[-1] = CascadeDict(seq[-1], cascaded_with) - - else: - obj.append(toplevel.name, toplevel.table_element) - last_array_of_tables = toplevel.name - - return obj diff --git a/pipenv/patched/contoml/file/test_cascadedict.py b/pipenv/patched/contoml/file/test_cascadedict.py deleted file mode 100644 index d692711e..00000000 --- a/pipenv/patched/contoml/file/test_cascadedict.py +++ /dev/null @@ -1,25 +0,0 @@ -from contoml.file.cascadedict import CascadeDict - - -def test_cascadedict(): - - d1 = {'a': 1, 'b': 2, 'c': 3} - d2 = {'b': 12, 'e': 4, 'f': 5} - - cascade = CascadeDict(d1, d2) - - # Test querying - assert cascade['a'] == 1 - assert cascade['b'] == 2 - assert cascade['c'] == 3 - assert cascade['e'] == 4 - assert cascade.keys() == {'a', 'b', 'c', 'e', 'f'} - assert set(cascade.items()) == {('a', 1), ('b', 2), ('c', 3), ('e', 4), ('f', 5)} - - # Test mutating - cascade['a'] = 11 - cascade['f'] = 'fff' - cascade['super'] = 'man' - assert d1['a'] == 11 - assert d1['super'] == 'man' - assert d1['f'] == 'fff' diff --git a/pipenv/patched/contoml/file/test_entries.py b/pipenv/patched/contoml/file/test_entries.py deleted file mode 100644 index 25584e82..00000000 --- a/pipenv/patched/contoml/file/test_entries.py +++ /dev/null @@ -1,20 +0,0 @@ -from prettytoml import parser, lexer -from contoml.file import toplevels - - -def test_entry_extraction(): - text = open('sample.toml').read() - elements = parser.parse_tokens(lexer.tokenize(text)) - - e = tuple(toplevels.identify(elements)) - - assert len(e) == 13 - assert isinstance(e[0], toplevels.AnonymousTable) - - -def test_entry_names(): - name_a = toplevels.Name(('super', 'sub1')) - name_b = toplevels.Name(('super', 'sub1', 'sub2', 'sub3')) - - assert name_b.is_prefixed_with(name_a) - assert name_b.without_prefix(name_a).sub_names == ('sub2', 'sub3') diff --git a/pipenv/patched/contoml/file/test_peekableit.py b/pipenv/patched/contoml/file/test_peekableit.py deleted file mode 100644 index 5c053a38..00000000 --- a/pipenv/patched/contoml/file/test_peekableit.py +++ /dev/null @@ -1,12 +0,0 @@ -from contoml.file.peekableit import PeekableIterator - - -def test_peekable_iterator(): - - peekable = PeekableIterator(i for i in (1, 2, 3, 4)) - - assert peekable.peek() == 1 - assert peekable.peek() == 1 - assert peekable.peek() == 1 - - assert [next(peekable), next(peekable), next(peekable), next(peekable)] == [1, 2, 3, 4] diff --git a/pipenv/patched/contoml/file/test_structurer.py b/pipenv/patched/contoml/file/test_structurer.py deleted file mode 100644 index b3ea4b4e..00000000 --- a/pipenv/patched/contoml/file/test_structurer.py +++ /dev/null @@ -1,41 +0,0 @@ -from prettytoml import lexer, parser -from contoml.file import toplevels -from prettytoml.parser import elementsanitizer -from contoml.file.structurer import NamedDict, structure -from prettytoml.parser.tokenstream import TokenStream - - -def test_NamedDict(): - - d = NamedDict() - - d[toplevels.Name(('super', 'sub1', 'sub2'))] = {'sub3': 12} - d[toplevels.Name(('super', 'sub1', 'sub2'))]['sub4'] = 42 - - assert d[toplevels.Name(('super', 'sub1', 'sub2', 'sub3'))] == 12 - assert d[toplevels.Name(('super', 'sub1', 'sub2', 'sub4'))] == 42 - - -def test_structure(): - tokens = lexer.tokenize(open('sample.toml').read()) - elements = elementsanitizer.sanitize(parser.parse_tokens(tokens)) - entries_ = tuple(toplevels.identify(elements)) - - s = structure(entries_) - - assert s['']['title'] == 'TOML Example' - assert s['owner']['name'] == 'Tom Preston-Werner' - assert s['database']['ports'][1] == 8001 - assert s['servers']['alpha']['dc'] == 'eqdc10' - assert s['clients']['data'][1][0] == 1 - assert s['clients']['key3'] == 'The quick brown fox jumps over the lazy dog.' - - assert s['fruit'][0]['name'] == 'apple' - assert s['fruit'][0]['physical']['color'] == 'red' - assert s['fruit'][0]['physical']['shape'] == 'round' - assert s['fruit'][0]['variety'][0]['name'] == 'red delicious' - assert s['fruit'][0]['variety'][1]['name'] == 'granny smith' - - assert s['fruit'][1]['name'] == 'banana' - assert s['fruit'][1]['variety'][0]['name'] == 'plantain' - assert s['fruit'][1]['variety'][0]['points'][2]['y'] == 4 diff --git a/pipenv/patched/contoml/file/toplevels.py b/pipenv/patched/contoml/file/toplevels.py deleted file mode 100644 index 64038072..00000000 --- a/pipenv/patched/contoml/file/toplevels.py +++ /dev/null @@ -1,142 +0,0 @@ -""" - Top-level entries in a TOML file. -""" - -from prettytoml import elements -from prettytoml.elements import TableElement, TableHeaderElement -from .peekableit import PeekableIterator - - -class TopLevel: - """ - A abstract top-level entry. - """ - - def __init__(self, names, table_element): - self._table_element = table_element - self._names = Name(names) - - @property - def table_element(self): - return self._table_element - - @property - def name(self): - """ - The distinct name of a table entry as an Name instance. - """ - return self._names - - -class Name: - - def __init__(self, names): - self._names = names - - @property - def sub_names(self): - return self._names - - def drop(self, n=0): - """ - Returns the name after dropping the first n entries of it. - """ - return Name(names=self._names[n:]) - - def is_prefixed_with(self, names): - if isinstance(names, Name): - return self.is_prefixed_with(names.sub_names) - - for i, name in enumerate(names): - if self._names[i] != name: - return False - return True - - def without_prefix(self, names): - if isinstance(names, Name): - return self.without_prefix(names.sub_names) - - for i, name in enumerate(names): - if name != self._names[i]: - return Name(self._names[i:]) - return Name(names=self.sub_names[len(names):]) - - @property - def is_qualified(self): - return len(self._names) > 1 - - def __str__(self): - return '.'.join(self.sub_names) - - def __hash__(self): - return hash(str(self)) - - def __eq__(self, other): - return str(self) == str(other) - - def __ne__(self, other): - return not self.__eq__(other) - - -class AnonymousTable(TopLevel): - - def __init__(self, table_element): - TopLevel.__init__(self, ('',), table_element) - - -class Table(TopLevel): - - def __init__(self, names, table_element): - TopLevel.__init__(self, names=names, table_element=table_element) - - -class ArrayOfTables(TopLevel): - - def __init__(self, names, table_element): - TopLevel.__init__(self, names=names, table_element=table_element) - - -def _validate_file_elements(file_elements): - pass - - -def identify(file_elements): - """ - Outputs an ordered sequence of instances of TopLevel types. - - Elements start with an optional TableElement, followed by zero or more pairs of (TableHeaderElement, TableElement). - """ - - if not file_elements: - return - - _validate_file_elements(file_elements) - - # An iterator over enumerate(the non-metadata) elements - iterator = PeekableIterator((element_i, element) for (element_i, element) in enumerate(file_elements) - if element.type != elements.TYPE_METADATA) - - try: - _, first_element = iterator.peek() - if isinstance(first_element, TableElement): - iterator.next() - yield AnonymousTable(first_element) - except KeyError: - pass - except StopIteration: - return - - for element_i, element in iterator: - - if not isinstance(element, TableHeaderElement): - continue - - # If TableHeader of a regular table, return Table following it - if not element.is_array_of_tables: - table_element_i, table_element = next(iterator) - yield Table(names=element.names, table_element=table_element) - - # If TableHeader of an array of tables, do your thing - else: - table_element_i, table_element = next(iterator) - yield ArrayOfTables(names=element.names, table_element=table_element) diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index f73ed5a6..4f3ee409 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -3,5 +3,4 @@ git+https://github.com/jumpscale7/python-consistent-toml.git#egg=contoml crayons==0.1.2 pipfile==0.0.2 pip-tools==3.1.0 -prettytoml==0.3 pip==18.1 diff --git a/pipenv/patched/prettytoml/LICENSE b/pipenv/patched/prettytoml/LICENSE deleted file mode 100644 index 116fa4e5..00000000 --- a/pipenv/patched/prettytoml/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Jumpscale - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/pipenv/patched/prettytoml/__init__.py b/pipenv/patched/prettytoml/__init__.py deleted file mode 100644 index d7310748..00000000 --- a/pipenv/patched/prettytoml/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -from ._version import VERSION - -__version__ = VERSION - - -def prettify(toml_text): - """ - Prettifies and returns the TOML file content provided. - """ - from .parser import parse_tokens - from .lexer import tokenize - from .prettifier import prettify as element_prettify - - tokens = tokenize(toml_text, is_top_level=True) - elements = parse_tokens(tokens) - prettified = element_prettify(elements) - return ''.join(pretty_element.serialized() for pretty_element in prettified) - - -def prettify_from_file(file_path): - """ - Reads, prettifies and returns the TOML file specified by the file_path. - """ - with open(file_path, 'r') as fp: - return prettify(fp.read()) diff --git a/pipenv/patched/prettytoml/_version.py b/pipenv/patched/prettytoml/_version.py deleted file mode 100644 index e0f15470..00000000 --- a/pipenv/patched/prettytoml/_version.py +++ /dev/null @@ -1 +0,0 @@ -VERSION = 'master' diff --git a/pipenv/patched/prettytoml/elements/__init__.py b/pipenv/patched/prettytoml/elements/__init__.py deleted file mode 100644 index ece21123..00000000 --- a/pipenv/patched/prettytoml/elements/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ - -""" - TOML file elements (a higher abstraction layer than individual lexical tokens). -""" - -from .traversal import TraversalMixin -from .errors import InvalidElementError -from .table import TableElement -from .tableheader import TableHeaderElement -from .common import TYPE_METADATA, TYPE_ATOMIC, TYPE_CONTAINER, TYPE_MARKUP - -from . import traversal -from . import factory diff --git a/pipenv/patched/prettytoml/elements/abstracttable.py b/pipenv/patched/prettytoml/elements/abstracttable.py deleted file mode 100644 index 6a2c174a..00000000 --- a/pipenv/patched/prettytoml/elements/abstracttable.py +++ /dev/null @@ -1,92 +0,0 @@ -try: - from collections.abc import Mapping -except ImportError: - from collections import Mapping - -from prettytoml.elements.common import ContainerElement -from prettytoml.elements import traversal - - -class AbstractTable(ContainerElement, traversal.TraversalMixin, Mapping): - """ - Common code for handling tables as key-value pairs with metadata elements sprinkled all over. - - Assumes input sub_elements are correct. - """ - - def __init__(self, sub_elements): - ContainerElement.__init__(self, sub_elements) - self._fallback = None - - def _enumerate_items(self): - """ - Returns ((key_index, key_element), (value_index, value_element)) for all the element key-value pairs. - """ - non_metadata = self._enumerate_non_metadata_sub_elements() - while True: - try: - yield next(non_metadata), next(non_metadata) - except StopIteration: - return - - def items(self): - for (key_i, key), (value_i, value) in self._enumerate_items(): - yield key.value, value.value - if self._fallback: - for key, value in self._fallback.items(): - yield key, value - - def keys(self): - return tuple(key for (key, _) in self.items()) - - def values(self): - return tuple(value for (_, value) in self.items()) - - def __len__(self): - return len(tuple(self._enumerate_items())) - - def __iter__(self): - return (key for key, _ in self.items()) - - def __contains__(self, item): - return item in self.keys() - - def _find_key_and_value(self, key): - """ - Returns (key_i, value_i) corresponding to the given key value. - - Raises KeyError if no matching key found. - """ - for (key_i, key_element), (value_i, value_element) in self._enumerate_items(): - if key_element.value == key: - return key_i, value_i - raise KeyError - - def __getitem__(self, item): - for key, value in self.items(): - if key == item: - return value - raise KeyError - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def set_fallback(self, fallback): - """ - Sets a fallback dict-like instance to be used to look up values after they are not found - in this instance. - """ - self._fallback = fallback - - @property - def primitive_value(self): - """ - Returns a primitive Python value without any formatting or markup metadata. - """ - return { - key: - value.primitive_value if hasattr(value, 'primitive_value') else value for key, value in self.items() - } diff --git a/pipenv/patched/prettytoml/elements/array.py b/pipenv/patched/prettytoml/elements/array.py deleted file mode 100644 index 36c648b9..00000000 --- a/pipenv/patched/prettytoml/elements/array.py +++ /dev/null @@ -1,136 +0,0 @@ -from prettytoml.elements import common, factory, traversal -from prettytoml.elements.common import Element, ContainerElement -from prettytoml.elements.factory import create_element -from prettytoml.elements.metadata import NewlineElement -from prettytoml.elements.errors import InvalidElementError - - -class ArrayElement(ContainerElement, traversal.TraversalMixin): - """ - A sequence-like container element containing other atomic elements or other containers. - - Implements list-like interface. - - Assumes input sub_elements are correct for an array element. - - Raises an InvalidElementError if contains heterogeneous values. - """ - - def __init__(self, sub_elements): - common.ContainerElement.__init__(self, sub_elements) - self._check_homogeneity() - - def _check_homogeneity(self): - if len(set(type(v) for v in self.primitive_value)) > 1: - raise InvalidElementError('Array should be homogeneous') - - def __len__(self): - return len(tuple(self._enumerate_non_metadata_sub_elements())) - - def __getitem__(self, i): - """ - Returns the ith entry, which can be a primitive value, a seq-lie, or a dict-like object. - """ - return self._find_value(i)[1].value - - def __setitem__(self, i, value): - value_i, _ = self._find_value(i) - new_element = value if isinstance(value, Element) else factory.create_element(value) - self._sub_elements = self.sub_elements[:value_i] + [new_element] + self.sub_elements[value_i+1:] - - @property - def value(self): - return self # self is a sequence-like value - - @property - def primitive_value(self): - """ - Returns a primitive Python value without any formatting or markup metadata. - """ - return list( - self[i].primitive_value if hasattr(self[i], 'primitive_value') - else self[i] - for i in range(len(self))) - - def __str__(self): - return "Array{}".format(self.primitive_value) - - def append(self, v): - new_entry = [create_element(v)] - - if self: # If not empty, we need a comma and whitespace prefix! - new_entry = [ - factory.create_operator_element(','), - factory.create_whitespace_element(), - ] + new_entry - - insertion_index = self._find_closing_square_bracket() - self._sub_elements = self._sub_elements[:insertion_index] + new_entry + \ - self._sub_elements[insertion_index:] - - def _find_value(self, i): - """ - Returns (value_index, value) of ith value in this sequence. - - Raises IndexError if not found. - """ - return tuple(self._enumerate_non_metadata_sub_elements())[i] - - def __delitem__(self, i): - value_i, value = self._find_value(i) - - begin, end = value_i, value_i+1 - - # Rules: - # 1. begin should be index to the preceding comma to the value - # 2. end should be index to the following comma, or the closing bracket - # 3. If no preceding comma found but following comma found then end should be the index of the following value - - preceding_comma = self._find_preceding_comma(value_i) - found_preceding_comma = preceding_comma >= 0 - if found_preceding_comma: - begin = preceding_comma - - following_comma = self._find_following_comma(value_i) - if following_comma >= 0: - if not found_preceding_comma: - end = self._find_following_non_metadata(following_comma) - else: - end = following_comma - else: - end = self._find_following_closing_square_bracket() - - self._sub_elements = self.sub_elements[:begin] + self._sub_elements[end:] - - @property - def is_multiline(self): - return any(isinstance(e, (NewlineElement)) for e in self.elements) - - def turn_into_multiline(self): - """ - Turns this array into a multi-line array with each element lying on its own line. - """ - if self.is_multiline: - return - - i = self._find_following_comma(-1) - - def next_entry_i(): - return self._find_following_non_metadata(i) - - def next_newline_i(): - return self._find_following_newline(i) - - def next_closing_bracket_i(): - return self._find_following_closing_square_bracket(i) - - def next_comma_i(): - return self._find_following_comma(i) - - while i < len(self.elements)-1: - if next_newline_i() < next_entry_i(): - self.elements.insert(i+1, factory.create_newline_element()) - if float('-inf') < next_comma_i() < next_closing_bracket_i(): - i = next_comma_i() - else: - i = next_closing_bracket_i() diff --git a/pipenv/patched/prettytoml/elements/atomic.py b/pipenv/patched/prettytoml/elements/atomic.py deleted file mode 100644 index 571810d9..00000000 --- a/pipenv/patched/prettytoml/elements/atomic.py +++ /dev/null @@ -1,52 +0,0 @@ -from ..tokens import py2toml, toml2py -from . import common -from prettytoml.util import is_dict_like, is_sequence_like -from .errors import InvalidElementError - - -class AtomicElement(common.TokenElement): - """ - An element containing a sequence of tokens representing a single atomic value that can be updated in place. - - Raises: - InvalidElementError: when passed an invalid sequence of tokens. - """ - - def __init__(self, _tokens): - common.TokenElement.__init__(self, _tokens, common.TYPE_ATOMIC) - - def _validate_tokens(self, _tokens): - if len([token for token in _tokens if not token.type.is_metadata]) != 1: - raise InvalidElementError('Tokens making up an AtomicElement must contain only one non-metadata token') - - def serialized(self): - return ''.join(token.source_substring for token in self.tokens) - - def _value_token_index(self): - """ - Finds the token where the value is stored. - """ - # TODO: memoize this value - for i, token in enumerate(self.tokens): - if not token.type.is_metadata: - return i - raise RuntimeError('could not find a value token') - - @property - def value(self): - """ - Returns a Python value contained in this atomic element. - """ - return toml2py.deserialize(self._tokens[self._value_token_index()]) - - @property - def primitive_value(self): - return self.value - - def set(self, value): - """ - Sets the contained value to the given one. - """ - assert (not is_sequence_like(value)) and (not is_dict_like(value)), 'the value must be an atomic primitive' - token_index = self._value_token_index() - self._tokens[token_index] = py2toml.create_primitive_token(value) diff --git a/pipenv/patched/prettytoml/elements/common.py b/pipenv/patched/prettytoml/elements/common.py deleted file mode 100644 index 7508047a..00000000 --- a/pipenv/patched/prettytoml/elements/common.py +++ /dev/null @@ -1,101 +0,0 @@ -from abc import abstractmethod - -TYPE_METADATA = 'element-metadata' -TYPE_ATOMIC = 'element-atomic' -TYPE_CONTAINER = 'element-container' -TYPE_MARKUP = 'element-markup' - -class Element: - """ - An Element: - - is one or more Token instances, or one or more other Element instances. Not both. - - knows how to serialize its value back to valid TOML code. - - A non-metadata Element is an Element that: - - knows how to deserialize its content into usable Python primitive, seq-like, or dict-like value. - - knows how to update its content from a Python primitive, seq-like, or dict-like value - while maintaining its formatting. - """ - - def __init__(self, _type): - self._type = _type - - @property - def type(self): - return self._type - - @abstractmethod - def serialized(self): - """ - TOML serialization of this element as str. - """ - raise NotImplementedError - - -class TokenElement(Element): - """ - An Element made up of tokens - """ - - def __init__(self, _tokens, _type): - Element.__init__(self, _type) - self._validate_tokens(_tokens) - self._tokens = list(_tokens) - - @property - def tokens(self): - return self._tokens - - @property - def first_token(self): - return self._tokens[0] - - @abstractmethod - def _validate_tokens(self, _tokens): - raise NotImplementedError - - def serialized(self): - return ''.join(token.source_substring for token in self._tokens) - - def __repr__(self): - return repr(self.tokens) - - @property - def primitive_value(self): - """ - Returns a primitive Python value without any formatting or markup metadata. - """ - raise NotImplementedError - - -class ContainerElement(Element): - """ - An Element containing exclusively other elements. - """ - - def __init__(self, sub_elements): - Element.__init__(self, TYPE_CONTAINER) - self._sub_elements = list(sub_elements) - - @property - def sub_elements(self): - return self._sub_elements - - @property - def elements(self): - return self.sub_elements - - def serialized(self): - return ''.join(element.serialized() for element in self.sub_elements) - - def __repr__(self): - return repr(self.primitive_value) - - @property - def primitive_value(self): - """ - Returns a primitive Python value without any formatting or markup metadata. - """ - raise NotImplementedError - - diff --git a/pipenv/patched/prettytoml/elements/errors.py b/pipenv/patched/prettytoml/elements/errors.py deleted file mode 100644 index 0fcf2e99..00000000 --- a/pipenv/patched/prettytoml/elements/errors.py +++ /dev/null @@ -1,13 +0,0 @@ - -class InvalidElementError(Exception): - """ - Raised by Element factories when the given sequence of tokens or sub-elements are invalid for the - specific type of Element being created. - """ - - def __init__(self, message): - self.message = message - - def __repr__(self): - return "InvalidElementError: {}".format(self.message) - diff --git a/pipenv/patched/prettytoml/elements/factory.py b/pipenv/patched/prettytoml/elements/factory.py deleted file mode 100644 index 177738db..00000000 --- a/pipenv/patched/prettytoml/elements/factory.py +++ /dev/null @@ -1,152 +0,0 @@ -import datetime -import functools -import six -from prettytoml import tokens -from prettytoml.tokens import py2toml -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import PunctuationElement, WhitespaceElement, NewlineElement -from prettytoml.elements.tableheader import TableHeaderElement -from prettytoml.util import join_with, is_sequence_like - - -def create_element(value, multiline_strings_allowed=True): - """ - Creates and returns the appropriate elements.Element instance from the given Python primitive, sequence-like, - or dict-like value. - """ - from prettytoml.elements.array import ArrayElement - - if isinstance(value, (int, float, bool, datetime.datetime, datetime.date) + six.string_types) or value is None: - primitive_token = py2toml.create_primitive_token(value, multiline_strings_allowed=multiline_strings_allowed) - return AtomicElement((primitive_token,)) - - elif isinstance(value, (list, tuple)): - preamble = [create_operator_element('[')] - postable = [create_operator_element(']')] - stuffing_elements = [create_element(v) for v in value] - spaced_stuffing = join_with(stuffing_elements, - separator=[create_operator_element(','), create_whitespace_element()]) - - return ArrayElement(preamble + spaced_stuffing + postable) - - elif isinstance(value, dict): - return create_inline_table(value, multiline_table=False, multiline_strings_allowed=multiline_strings_allowed) - - else: - raise RuntimeError('Value type unaccounted for: {} of type {}'.format(value, type(value))) - - -def create_inline_table(from_dict, multiline_table=False, multiline_strings_allowed=True): - """ - Creates an InlineTable element from the given dict instance. - """ - - from prettytoml.elements.inlinetable import InlineTableElement - - preamble = [create_operator_element('{')] - postable = [create_operator_element('}')] - - stuffing_elements = ( - ( - create_string_element(k, bare_allowed=True), - create_whitespace_element(), - create_operator_element('='), - create_whitespace_element(), - create_element(v, multiline_strings_allowed=False) - ) for (k, v) in from_dict.items()) - - pair_separator = [create_operator_element(','), - create_newline_element() if multiline_table else create_whitespace_element()] - spaced_elements = join_with(stuffing_elements, separator=pair_separator) - - return InlineTableElement(preamble + spaced_elements + postable) - - -def create_string_element(value, bare_allowed=False): - """ - Creates and returns an AtomicElement wrapping a string value. - """ - return AtomicElement((py2toml.create_string_token(value, bare_allowed),)) - - -def create_operator_element(operator): - """ - Creates a PunctuationElement instance containing an operator token of the specified type. The operator - should be a TOML source str. - """ - operator_type_map = { - ',': tokens.TYPE_OP_COMMA, - '=': tokens.TYPE_OP_ASSIGNMENT, - '[': tokens.TYPE_OP_SQUARE_LEFT_BRACKET, - ']': tokens.TYPE_OP_SQUARE_RIGHT_BRACKET, - '[[': tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET, - ']]': tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET, - '{': tokens.TYPE_OP_CURLY_LEFT_BRACKET, - '}': tokens.TYPE_OP_CURLY_RIGHT_BRACKET, - } - - ts = (tokens.Token(operator_type_map[operator], operator),) - return PunctuationElement(ts) - - -def create_newline_element(): - """ - Creates and returns a single NewlineElement. - """ - ts = (tokens.Token(tokens.TYPE_NEWLINE, '\n'),) - return NewlineElement(ts) - - -def create_whitespace_element(length=1, char=' '): - """ - Creates and returns a WhitespaceElement containing spaces. - """ - ts = (tokens.Token(tokens.TYPE_WHITESPACE, char),) * length - return WhitespaceElement(ts) - - -def create_table_header_element(names): - - name_elements = [] - - if isinstance(names, six.string_types): - name_elements = [py2toml.create_string_token(names, bare_string_allowed=True)] - else: - for (i, name) in enumerate(names): - name_elements.append(py2toml.create_string_token(name, bare_string_allowed=True)) - if i < (len(names)-1): - name_elements.append(py2toml.operator_token(tokens.TYPE_OPT_DOT)) - - return TableHeaderElement( - [py2toml.operator_token(tokens.TYPE_OP_SQUARE_LEFT_BRACKET)] + name_elements + - [py2toml.operator_token(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET), py2toml.operator_token(tokens.TYPE_NEWLINE)], - ) - - -def create_array_of_tables_header_element(name): - return TableHeaderElement(( - py2toml.operator_token(tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET), - py2toml.create_string_token(name, bare_string_allowed=True), - py2toml.operator_token(tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET), - py2toml.operator_token(tokens.TYPE_NEWLINE), - )) - - -def create_table(dict_value): - """ - Creates a TableElement out of a dict instance. - """ - from prettytoml.elements.table import TableElement - - if not isinstance(dict_value, dict): - raise ValueError('input must be a dict instance.') - - table_element = TableElement([create_newline_element()]) - for k, v in dict_value.items(): - table_element[k] = create_element(v) - - return table_element - - -def create_multiline_string(text, maximum_line_length): - return AtomicElement(_tokens=[py2toml.create_multiline_string(text, maximum_line_length)]) diff --git a/pipenv/patched/prettytoml/elements/inlinetable.py b/pipenv/patched/prettytoml/elements/inlinetable.py deleted file mode 100644 index 7b985fc1..00000000 --- a/pipenv/patched/prettytoml/elements/inlinetable.py +++ /dev/null @@ -1,78 +0,0 @@ -from prettytoml.elements import factory, abstracttable -from prettytoml.elements.common import Element - - -class InlineTableElement(abstracttable.AbstractTable): - """ - An Element containing key-value pairs, representing an inline table. - - Implements dict-like interface. - - Assumes input sub_elements are correct for an inline table element. - """ - - def __init__(self, sub_elements): - abstracttable.AbstractTable.__init__(self, sub_elements) - - def __setitem__(self, key, value): - - new_element = value if isinstance(value, Element) else factory.create_element(value) - - try: - - key_i, value_i = self._find_key_and_value(key) - # Found, then replace the value element with a new one - self._sub_elements = self.sub_elements[:value_i] + [new_element] + self.sub_elements[value_i+1:] - - except KeyError: # Key does not exist, adding anew! - - new_entry = [ - factory.create_string_element(key, bare_allowed=True), - factory.create_whitespace_element(), - factory.create_operator_element('='), - factory.create_whitespace_element(), - new_element, - ] - - if self: # If not empty - new_entry = [ - factory.create_operator_element(','), - factory.create_whitespace_element(), - ] + new_entry - - insertion_index = self._find_closing_curly_bracket() - self._sub_elements = self.sub_elements[:insertion_index] + new_entry + self.sub_elements[insertion_index:] - - def __delitem__(self, key): - - key_i, value_i = self._find_key_and_value(key) - - begin, end = key_i, value_i+1 - - # Rules: - # 1. begin should be index to the preceding comma to the key - # 2. end should be index to the following comma, or the closing bracket - # 3. If no preceding comma found but following comma found then end should be the index of the following key - - preceding_comma = self._find_preceding_comma(begin) - found_preceding_comma = preceding_comma >= 0 - if found_preceding_comma: - begin = preceding_comma - - following_comma = self._find_following_comma(value_i) - if following_comma >= 0: - if not found_preceding_comma: - end = self._find_following_non_metadata(following_comma) - else: - end = following_comma - else: - end = self._find_closing_curly_bracket() - - self._sub_elements = self.sub_elements[:begin] + self.sub_elements[end:] - - def multiline_equivalent(self): - return factory.create_inline_table(self.primitive_value, multiline_table=True, multiline_strings_allowed=True) - - @property - def value(self): - return self # self is a dict-like value that is perfectly usable diff --git a/pipenv/patched/prettytoml/elements/metadata.py b/pipenv/patched/prettytoml/elements/metadata.py deleted file mode 100644 index d5ee1061..00000000 --- a/pipenv/patched/prettytoml/elements/metadata.py +++ /dev/null @@ -1,80 +0,0 @@ -from prettytoml import tokens -from prettytoml.elements import common -from .errors import InvalidElementError - - -class WhitespaceElement(common.TokenElement): - """ - An element that contains tokens of whitespace - """ - - def __init__(self, _tokens): - common.TokenElement.__init__(self, _tokens, common.TYPE_METADATA) - - def _validate_tokens(self, _tokens): - for token in _tokens: - if token.type != tokens.TYPE_WHITESPACE: - raise InvalidElementError('Tokens making up a WhitespaceElement must all be whitespace') - - @property - def length(self): - """ - The whitespace length of this element - """ - return len(self.tokens) - - -class NewlineElement(common.TokenElement): - """ - An element containing newline tokens - - Raises: - InvalidElementError: when passed an invalid sequence of tokens. - """ - - def __init__(self, _tokens): - common.TokenElement.__init__(self, _tokens, common.TYPE_METADATA) - - def _validate_tokens(self, _tokens): - for token in _tokens: - if token.type != tokens.TYPE_NEWLINE: - raise InvalidElementError('Tokens making a NewlineElement must all be newlines') - - -class CommentElement(common.TokenElement): - """ - An element containing a single comment token followed by a newline. - - Raises: - InvalidElementError: when passed an invalid sequence of tokens. - """ - - def __init__(self, _tokens): - common.TokenElement.__init__(self, _tokens, common.TYPE_METADATA) - - def _validate_tokens(self, _tokens): - if len(_tokens) != 2 or _tokens[0].type != tokens.TYPE_COMMENT or _tokens[1].type != tokens.TYPE_NEWLINE: - raise InvalidElementError('CommentElement needs one comment token followed by one newline token') - - -class PunctuationElement(common.TokenElement): - """ - An element containing a single punctuation token. - - Raises: - InvalidElementError: when passed an invalid sequence of tokens. - """ - - def __init__(self, _tokens): - common.TokenElement.__init__(self, _tokens, common.TYPE_METADATA) - - @property - def token(self): - """ - Returns the token contained in this Element. - """ - return self.tokens[0] - - def _validate_tokens(self, _tokens): - if not _tokens or not tokens.is_operator(_tokens[0]): - raise InvalidElementError('PunctuationElement must be made of only a single operator token') diff --git a/pipenv/patched/prettytoml/elements/table.py b/pipenv/patched/prettytoml/elements/table.py deleted file mode 100644 index cdc3ed4c..00000000 --- a/pipenv/patched/prettytoml/elements/table.py +++ /dev/null @@ -1,122 +0,0 @@ -from prettytoml.elements import abstracttable, factory -from prettytoml.elements.errors import InvalidElementError -from prettytoml.elements.common import Element -from prettytoml.elements.metadata import CommentElement, NewlineElement, WhitespaceElement -from . import common - - -class TableElement(abstracttable.AbstractTable): - """ - An Element containing an unnamed top-level table. - - Implements dict-like interface. - - Assumes input sub_elements are correct. - - Raises InvalidElementError on duplicate keys. - """ - - def __init__(self, sub_elements): - abstracttable.AbstractTable.__init__(self, sub_elements) - - self._check_for_duplicate_keys() - - def _check_for_duplicate_keys(self): - if len(set(self.keys())) < len(self.keys()): - raise InvalidElementError('Duplicate keys found') - - def __setitem__(self, key, value): - if key in self: - self._update(key, value) - else: - self._insert(key, value) - - def _update(self, key, value): - _, value_i = self._find_key_and_value(key) - self._sub_elements[value_i] = value if isinstance(value, Element) else factory.create_element(value) - - def _find_insertion_index(self): - """ - Returns the self.sub_elements index in which new entries should be inserted. - """ - - non_metadata_elements = tuple(self._enumerate_non_metadata_sub_elements()) - - if not non_metadata_elements: - return 0 - - last_entry_i = non_metadata_elements[-1][0] - following_newline_i = self._find_following_line_terminator(last_entry_i) - - return following_newline_i + 1 - - def _detect_indentation_size(self): - """ - Detects the level of indentation used in this table. - """ - - def lines(): - # Returns a sequence of sequences of elements belonging to each line - start = 0 - for i, element in enumerate(self.elements): - if isinstance(element, (CommentElement, NewlineElement)): - yield self.elements[start:i+1] - start = i+1 - - def indentation(line): - # Counts the number of whitespace tokens at the beginning of this line - try: - first_non_whitespace_i = next(i for (i, e) in enumerate(line) if not isinstance(e, WhitespaceElement)) - return sum(space.length for space in line[:first_non_whitespace_i]) - except StopIteration: - return 0 - - def is_empty_line(line): - return all(e.type == common.TYPE_METADATA for e in line) - - try: - return min(indentation(line) for line in lines() if len(line) > 1 and not is_empty_line(line)) - except ValueError: # Raised by ValueError when no matching lines found - return 0 - - def _insert(self, key, value): - - value_element = value if isinstance(value, Element) else factory.create_element(value) - - indentation_size = self._detect_indentation_size() - indentation = [factory.create_whitespace_element(self._detect_indentation_size())] if indentation_size else [] - - inserted_elements = indentation + [ - factory.create_string_element(key, bare_allowed=True), - factory.create_whitespace_element(), - factory.create_operator_element('='), - factory.create_whitespace_element(), - value_element, - factory.create_newline_element(), - ] - - insertion_index = self._find_insertion_index() - - self._sub_elements = \ - self.sub_elements[:insertion_index] + inserted_elements + self.sub_elements[insertion_index:] - - def __delitem__(self, key): - begin, _ = self._find_key_and_value(key) - preceding_newline = self._find_preceding_newline(begin) - if preceding_newline >= 0: - begin = preceding_newline - end = self._find_following_line_terminator(begin) - if end < 0: - end = len(tuple(self._sub_elements)) - self._sub_elements = self.sub_elements[:begin] + self.sub_elements[end:] - - def pop(self, key): - v = self[key] - del self[key] - return v - - def value(self): - return self - - def __str__(self): - return str(self.primitive_value) diff --git a/pipenv/patched/prettytoml/elements/tableheader.py b/pipenv/patched/prettytoml/elements/tableheader.py deleted file mode 100644 index eacd88b9..00000000 --- a/pipenv/patched/prettytoml/elements/tableheader.py +++ /dev/null @@ -1,95 +0,0 @@ -from prettytoml import tokens -from prettytoml.tokens import toml2py -from prettytoml.elements import common -from prettytoml.elements.common import Element, TokenElement -from prettytoml.elements.errors import InvalidElementError - -_opening_bracket_types = (tokens.TYPE_OP_SQUARE_LEFT_BRACKET, tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET) -_closing_bracket_types = (tokens.TYPE_OP_SQUARE_RIGHT_BRACKET, tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET) -_name_types = ( - tokens.TYPE_BARE_STRING, - tokens.TYPE_LITERAL_STRING, - tokens.TYPE_STRING, -) - - -class TableHeaderElement(TokenElement): - """ - An element containing opening and closing single and double square brackets, strings and dots and ending with - a newline. - - Raises InvalidElementError. - """ - - def __init__(self, _tokens): - TokenElement.__init__(self, _tokens, common.TYPE_MARKUP) - self._names = tuple(toml2py.deserialize(token) for token in self._tokens if token.type in _name_types) - - @property - def is_array_of_tables(self): - opening_bracket = next(token for i, token in enumerate(self._tokens) if token.type in _opening_bracket_types) - return opening_bracket.type == tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET - - @property - def names(self): - """ - Returns a sequence of string names making up this table header name. - """ - return self._names - - def has_name_prefix(self, names): - """ - Returns True if the header names is prefixed by the given sequence of names. - """ - for i, name in enumerate(names): - if self.names[i] != name: - return False - return True - - def serialized(self): - return ''.join(token.source_substring for token in self._tokens) - - def is_named(self, names): - """ - Returns True if the given name sequence matches the full name of this header. - """ - return tuple(names) == self.names - - def _validate_tokens(self, _tokens): - - opening_bracket_i = next((i for i, token in enumerate(_tokens) - if token.type in _opening_bracket_types), float('-inf')) - - if opening_bracket_i < 0: - raise InvalidElementError('Expected an opening bracket') - - _tokens = _tokens[opening_bracket_i+1:] - first_name_i = next((i for i, token in enumerate(_tokens) if token.type in _name_types), float('-inf')) - if first_name_i < 0: - raise InvalidElementError('Expected a table header name') - - _tokens = _tokens[first_name_i+1:] - - while True: - - next_dot_i = next((i for i, token in enumerate(_tokens) if token.type == tokens.TYPE_OPT_DOT), - float('-inf')) - if next_dot_i < 0: - break - - _tokens = _tokens[next_dot_i+1:] - - next_name_i = next((i for i, token in enumerate(_tokens) if token.type in _name_types), float('-inf')) - if next_name_i < 0: - raise InvalidElementError('Expected a name after the dot') - - _tokens = _tokens[next_name_i+1:] - - closing_bracket_i = next((i for i, token in enumerate(_tokens) if token.type in _closing_bracket_types), - float('-inf')) - - if closing_bracket_i < 0: - raise InvalidElementError('Expected a closing bracket') - - if _tokens[-1].type != tokens.TYPE_NEWLINE: - raise InvalidElementError('Must end with a newline') diff --git a/pipenv/patched/prettytoml/elements/test_array.py b/pipenv/patched/prettytoml/elements/test_array.py deleted file mode 100644 index 3ccc98b5..00000000 --- a/pipenv/patched/prettytoml/elements/test_array.py +++ /dev/null @@ -1,67 +0,0 @@ -import pytest -from prettytoml import lexer -from prettytoml.elements.array import ArrayElement -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import PunctuationElement, WhitespaceElement, NewlineElement - - -def test_array_element(): - tokens = tuple(lexer.tokenize('[4, 8, 42, \n 23, 15]')) - assert len(tokens) == 17 - sub_elements = ( - PunctuationElement(tokens[:1]), - - AtomicElement(tokens[1:2]), - PunctuationElement(tokens[2:3]), - WhitespaceElement(tokens[3:4]), - - AtomicElement(tokens[4:5]), - PunctuationElement(tokens[5:6]), - WhitespaceElement(tokens[6:7]), - - AtomicElement(tokens[7:8]), - PunctuationElement(tokens[8:9]), - WhitespaceElement(tokens[9:10]), - NewlineElement(tokens[10:11]), - WhitespaceElement(tokens[11:12]), - - AtomicElement(tokens[12:13]), - PunctuationElement(tokens[13:14]), - - WhitespaceElement(tokens[14:15]), - AtomicElement(tokens[15:16]), - PunctuationElement(tokens[16:17]) - ) - - array_element = ArrayElement(sub_elements) - - # Test length - assert len(array_element) == 5 - - # Test getting a value - assert array_element[0] == 4 - assert array_element[1] == 8 - assert array_element[2] == 42 - assert array_element[3] == 23 - assert array_element[-1] == 15 - - # Test assignment with a negative index - array_element[-1] = 12 - - # Test persistence of formatting - assert '[4, 8, 42, \n 23, 12]' == array_element.serialized() - - # Test raises IndexError on invalid index - with pytest.raises(IndexError) as _: - print(array_element[5]) - - # Test appending a new value - array_element.append(77) - assert '[4, 8, 42, \n 23, 12, 77]' == array_element.serialized() - - # Test deleting a value - del array_element[3] - assert '[4, 8, 42, 12, 77]' == array_element.serialized() - - # Test primitive_value - assert [4, 8, 42, 12, 77] == array_element.primitive_value diff --git a/pipenv/patched/prettytoml/elements/test_atomic.py b/pipenv/patched/prettytoml/elements/test_atomic.py deleted file mode 100644 index 940ddd27..00000000 --- a/pipenv/patched/prettytoml/elements/test_atomic.py +++ /dev/null @@ -1,9 +0,0 @@ -from prettytoml import lexer -from prettytoml.elements.atomic import AtomicElement - - -def test_atomic_element(): - element = AtomicElement(tuple(lexer.tokenize(' \t 42 '))) - assert element.value == 42 - element.set(23) - assert element.serialized() == ' \t 23 ' diff --git a/pipenv/patched/prettytoml/elements/test_common.py b/pipenv/patched/prettytoml/elements/test_common.py deleted file mode 100644 index 9f5dd4c8..00000000 --- a/pipenv/patched/prettytoml/elements/test_common.py +++ /dev/null @@ -1,89 +0,0 @@ -from prettytoml import tokens, lexer -from prettytoml.elements import traversal -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import NewlineElement, PunctuationElement, WhitespaceElement, CommentElement -from prettytoml.elements.table import TableElement -from prettytoml.elements.tableheader import TableHeaderElement - -atomic_token_types = ( - tokens.TYPE_INTEGER, - tokens.TYPE_FLOAT, - tokens.TYPE_BARE_STRING, - tokens.TYPE_STRING, - tokens.TYPE_LITERAL_STRING, - tokens.TYPE_MULTILINE_STRING, - tokens.TYPE_MULTILINE_LITERAL_STRING, -) - -punctuation_token_types = ( - tokens.TYPE_OPT_DOT, - tokens.TYPE_OP_CURLY_LEFT_BRACKET, - tokens.TYPE_OP_SQUARE_LEFT_BRACKET, - tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET, - tokens.TYPE_OP_SQUARE_RIGHT_BRACKET, - tokens.TYPE_OP_CURLY_RIGHT_BRACKET, - tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET, - tokens.TYPE_OP_ASSIGNMENT, -) - -def primitive_token_to_primitive_element(token): - if token.type == tokens.TYPE_NEWLINE: - return NewlineElement((token,)) - elif token.type in atomic_token_types: - return AtomicElement((token,)) - elif token.type == tokens.TYPE_NEWLINE: - return NewlineElement((token,)) - elif token.type in punctuation_token_types: - return PunctuationElement((token,)) - elif token.type == tokens.TYPE_WHITESPACE: - return WhitespaceElement((token,)) - elif token.type == tokens.TYPE_COMMENT: - return CommentElement((token,)) - else: - raise RuntimeError("{} has no mapped primitive element".format(token)) - - -def primitive_tokens_to_primitive_elements(tokens): - return list(map(primitive_token_to_primitive_element, tokens)) - - -def dummy_file_elements(): - tokens_ = tuple(lexer.tokenize(""" -name = fawzy -another_name=another_fawzy - -[details] -id= 42 -section =fourth - -[[person]] -personname= lefawzy -dest=north - -[[person]] -dest=south -personname=lafawzy - -[details.extended] -number = 313 -type =complex""")) - - elements = \ - [TableElement(primitive_tokens_to_primitive_elements(tokens_[:12]))] + \ - [TableHeaderElement(tokens_[12:16])] + \ - [TableElement(primitive_tokens_to_primitive_elements(tokens_[16:25]))] + \ - [TableHeaderElement(tokens_[25:31])] + \ - [TableElement(primitive_tokens_to_primitive_elements(tokens_[31:39]))] + \ - [TableHeaderElement(tokens_[39:45])] + \ - [TableElement(primitive_tokens_to_primitive_elements(tokens_[45:53]))] + \ - [TableHeaderElement(tokens_[53:60])] + \ - [TableElement(primitive_tokens_to_primitive_elements(tokens_[60:]))] - - return elements - - -class DummyFile(traversal.TraversalMixin): - - @property - def elements(self): - return dummy_file_elements() diff --git a/pipenv/patched/prettytoml/elements/test_factory.py b/pipenv/patched/prettytoml/elements/test_factory.py deleted file mode 100644 index 08a42883..00000000 --- a/pipenv/patched/prettytoml/elements/test_factory.py +++ /dev/null @@ -1,22 +0,0 @@ -from collections import OrderedDict -from prettytoml.elements import factory -from prettytoml.elements.array import ArrayElement -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.inlinetable import InlineTableElement - - -def test_creating_elements(): - - atomic = factory.create_element(42) - assert isinstance(atomic, AtomicElement) - assert atomic.value == 42 - - seq = factory.create_element(['a', 'p', 'p', 'l', 'e']) - assert isinstance(seq, ArrayElement) - assert seq.serialized() == '["a", "p", "p", "l", "e"]' - assert ''.join(seq.primitive_value) == 'apple' - - mapping = factory.create_element(OrderedDict((('one', 1), ('two', 2)))) - assert isinstance(mapping, InlineTableElement) - assert mapping.serialized() == '{one = 1, two = 2}' - diff --git a/pipenv/patched/prettytoml/elements/test_inlinetable.py b/pipenv/patched/prettytoml/elements/test_inlinetable.py deleted file mode 100644 index 3c663873..00000000 --- a/pipenv/patched/prettytoml/elements/test_inlinetable.py +++ /dev/null @@ -1,52 +0,0 @@ -from prettytoml import lexer -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.inlinetable import InlineTableElement -from prettytoml.elements.metadata import PunctuationElement, WhitespaceElement - - -def test_inline_table(): - tokens = tuple(lexer.tokenize('{ name= "first", id=42}')) - - elements = ( - PunctuationElement(tokens[:1]), - WhitespaceElement(tokens[1:2]), - AtomicElement(tokens[2:3]), - PunctuationElement(tokens[3:4]), - WhitespaceElement(tokens[4:5]), - AtomicElement(tokens[5:6]), - PunctuationElement(tokens[6:7]), - WhitespaceElement(tokens[7:8]), - AtomicElement(tokens[8:9]), - PunctuationElement(tokens[9:10]), - AtomicElement(tokens[10:11]), - PunctuationElement(tokens[11:12]) - ) - - table = InlineTableElement(elements) - - assert table['name'] == 'first' - assert table['id'] == 42 - - table['name'] = 'fawzy' - table['nickname'] = 'nickfawzy' - - assert set(table.items()) == {('name', 'fawzy'), ('id', 42), ('nickname', 'nickfawzy')} - - assert table.serialized() == '{ name= "fawzy", id=42, nickname = "nickfawzy"}' - - del table['name'] - - assert table.serialized() == '{ id=42, nickname = "nickfawzy"}' - - del table['nickname'] - - assert table.serialized() == '{ id=42}' - - del table['id'] - - assert table.serialized() == '{ }' - - table['item1'] = 11 - table['item2'] = 22 - - assert table.serialized() == '{ item1 = 11, item2 = 22}' diff --git a/pipenv/patched/prettytoml/elements/test_metadata.py b/pipenv/patched/prettytoml/elements/test_metadata.py deleted file mode 100644 index 6e49fedd..00000000 --- a/pipenv/patched/prettytoml/elements/test_metadata.py +++ /dev/null @@ -1,25 +0,0 @@ -from prettytoml import lexer -from prettytoml.elements.metadata import WhitespaceElement, NewlineElement, CommentElement, PunctuationElement - - -def test_whitespace_element(): - element = WhitespaceElement(tuple(lexer.tokenize(' \t '))) - assert element.serialized() == ' \t ' - - -def test_newline_element(): - element = NewlineElement(tuple(lexer.tokenize('\n\n\n'))) - assert element.serialized() == '\n\n\n' - - -def test_comment_element(): - element = CommentElement(tuple(lexer.tokenize('# This is my insightful remark\n'))) - assert element.serialized() == '# This is my insightful remark\n' - - -def test_punctuation_element(): - PunctuationElement(tuple(lexer.tokenize('['))) - PunctuationElement(tuple(lexer.tokenize('[['))) - PunctuationElement(tuple(lexer.tokenize('.'))) - PunctuationElement(tuple(lexer.tokenize(']'))) - PunctuationElement(tuple(lexer.tokenize(']]'))) diff --git a/pipenv/patched/prettytoml/elements/test_table.py b/pipenv/patched/prettytoml/elements/test_table.py deleted file mode 100644 index ef0dba81..00000000 --- a/pipenv/patched/prettytoml/elements/test_table.py +++ /dev/null @@ -1,59 +0,0 @@ -from prettytoml import lexer -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import WhitespaceElement, PunctuationElement, NewlineElement, CommentElement -from prettytoml.elements.table import TableElement - - -def test_table(): - - initial_toml = """name = "first" -id=42 # My id - - -""" - - tokens = tuple(lexer.tokenize(initial_toml)) - - elements = ( - AtomicElement(tokens[:1]), - WhitespaceElement(tokens[1:2]), - PunctuationElement(tokens[2:3]), - WhitespaceElement(tokens[3:4]), - AtomicElement(tokens[4:5]), - NewlineElement(tokens[5:6]), - - AtomicElement(tokens[6:7]), - PunctuationElement(tokens[7:8]), - AtomicElement(tokens[8:9]), - WhitespaceElement(tokens[9:10]), - CommentElement(tokens[10:12]), - - NewlineElement(tokens[12:13]), - NewlineElement(tokens[13:14]), - ) - - table = TableElement(elements) - - assert set(table.items()) == {('name', 'first'), ('id', 42)} - - assert table['name'] == 'first' - assert table['id'] == 42 - - table['relation'] = 'another' - - assert set(table.items()) == {('name', 'first'), ('id', 42), ('relation', 'another')} - - table['name'] = 'fawzy' - - assert set(table.items()) == {('name', 'fawzy'), ('id', 42), ('relation', 'another')} - - expected_toml = """name = "fawzy" -id=42 # My id -relation = "another" - - -""" - - assert table.serialized() == expected_toml - - diff --git a/pipenv/patched/prettytoml/elements/test_tableheader.py b/pipenv/patched/prettytoml/elements/test_tableheader.py deleted file mode 100644 index 67b1ded4..00000000 --- a/pipenv/patched/prettytoml/elements/test_tableheader.py +++ /dev/null @@ -1,12 +0,0 @@ -from prettytoml import lexer -from prettytoml.elements.tableheader import TableHeaderElement - - -def test_tableheader(): - tokens = tuple(lexer.tokenize('\n\t [[personal. information.details]] \n')) - element = TableHeaderElement(tokens) - - assert element.is_array_of_tables - assert ('personal', 'information', 'details') == element.names - - assert element.has_name_prefix(('personal', 'information')) diff --git a/pipenv/patched/prettytoml/elements/test_traversal.py b/pipenv/patched/prettytoml/elements/test_traversal.py deleted file mode 100644 index 2f002571..00000000 --- a/pipenv/patched/prettytoml/elements/test_traversal.py +++ /dev/null @@ -1,18 +0,0 @@ -from prettytoml.elements.test_common import DummyFile - - -def test_traversal(): - dummy_file = DummyFile() - - assert dummy_file._find_following_table_header(-1) == 1 - assert dummy_file._find_following_table_header(1) == 3 - assert dummy_file._find_following_table_header(3) == 5 - assert dummy_file._find_following_table_header(5) == 7 - assert dummy_file._find_following_table_header(7) < 0 - - assert dummy_file._find_preceding_table(30) == 8 - assert dummy_file._find_preceding_table(8) == 6 - assert dummy_file._find_preceding_table(6) == 4 - assert dummy_file._find_preceding_table(4) == 2 - assert dummy_file._find_preceding_table(2) == 0 - assert dummy_file._find_preceding_table(0) < 0 diff --git a/pipenv/patched/prettytoml/elements/traversal/__init__.py b/pipenv/patched/prettytoml/elements/traversal/__init__.py deleted file mode 100644 index c93506e2..00000000 --- a/pipenv/patched/prettytoml/elements/traversal/__init__.py +++ /dev/null @@ -1,175 +0,0 @@ -from prettytoml import tokens -from prettytoml.elements import common -from prettytoml.elements.metadata import PunctuationElement, NewlineElement -from prettytoml.elements.traversal import predicates - - -class TraversalMixin: - """ - A mix-in that provides convenient sub-element traversal to any class with - an `elements` member that is a sequence of Element instances - """ - - def __find_following_element(self, index, predicate): - """ - Finds and returns the index of element in self.elements that evaluates the given predicate to True - and whose index is higher than the given index, or returns -Infinity on failure. - """ - return find_following(self.elements, predicate, index) - - def __find_preceding_element(self, index, predicate): - """ - Finds and returns the index of the element in self.elements that evaluates the given predicate to True - and whose index is lower than the given index. - """ - i = find_previous(self.elements, predicate, index) - if i == float('inf'): - return float('-inf') - return i - - def __must_find_following_element(self, predicate): - """ - Finds and returns the index to the element in self.elements that evaluatest the predicate to True, or raises - an error. - """ - i = self.__find_following_element(-1, predicate) - if i < 0: - raise RuntimeError('Could not find non-optional element') - return i - - def _enumerate_non_metadata_sub_elements(self): - """ - Returns a sequence of of (index, sub_element) of the non-metadata sub-elements. - """ - return ((i, element) for i, element in enumerate(self.elements) if element.type != common.TYPE_METADATA) - - def _find_preceding_comma(self, index): - """ - Returns the index of the preceding comma element to the given index, or -Infinity. - """ - return self.__find_preceding_element(index, predicates.op_comma) - - def _find_following_comma(self, index): - """ - Returns the index of the following comma element after the given index, or -Infinity. - """ - def predicate(element): - return isinstance(element, PunctuationElement) and element.token.type == tokens.TYPE_OP_COMMA - return self.__find_following_element(index, predicate) - - def _find_following_newline(self, index): - """ - Returns the index of the following newline element after the given index, or -Infinity. - """ - return self.__find_following_element(index, lambda e: isinstance(e, NewlineElement)) - - def _find_following_comment(self, index): - """ - Returns the index of the following comment element after the given index, or -Infinity. - """ - return self.__find_following_element(index, predicates.comment) - - def _find_following_line_terminator(self, index): - """ - Returns the index of the following comment or newline element after the given index, or -Infinity. - """ - following_comment = self._find_following_comment(index) - following_newline = self._find_following_newline(index) - - if following_comment == float('-inf'): - return following_newline - if following_newline == float('-inf'): - return following_comment - - if following_newline < following_comment: - return following_newline - else: - return following_comment - - def _find_preceding_newline(self, index): - """ - Returns the index of the preceding newline element to the given index, or -Infinity. - """ - return self.__find_preceding_element(index, predicates.newline) - - def _find_following_non_metadata(self, index): - """ - Returns the index to the following non-metadata element after the given index, or -Infinity. - """ - return self.__find_following_element(index, predicates.non_metadata) - - def _find_closing_square_bracket(self): - """ - Returns the index to the closing square bracket, or raises an Error. - """ - - return self.__must_find_following_element(predicates.closing_square_bracket) - - def _find_following_opening_square_bracket(self, index): - """ - Returns the index to the opening square bracket, or -Infinity. - """ - return self.__find_following_element(index, predicates.opening_square_bracket) - - def _find_following_closing_square_bracket(self, index): - """ - Returns the index to the closing square bracket, or -Infinity. - """ - return self.__find_following_element(index, predicates.closing_square_bracket) - - def _find_following_table(self, index): - """ - Returns the index to the next TableElement after the specified index, or -Infinity. - """ - return self.__find_following_element(index, predicates.table) - - def _find_preceding_table(self, index): - """ - Returns the index to the preceding TableElement to the specified index, or -Infinity. - """ - return self.__find_preceding_element(index,predicates.table) - - def _find_closing_curly_bracket(self): - """ - Returns the index to the closing curly bracket, or raises an Error. - """ - def predicate(element): - return isinstance(element, PunctuationElement) and element.token.type == tokens.TYPE_OP_CURLY_RIGHT_BRACKET - return self.__must_find_following_element(predicate) - - def _find_following_table_header(self, index): - """ - Returns the index to the table header after the given element index, or -Infinity. - """ - return self.__find_following_element(index, predicates.table_header) - - -def find_following(element_seq, predicate, index=None): - """ - Finds and returns the index of the next element fulfilling the specified predicate after the specified - index, or -Infinity. - - Starts searching linearly from the start_from index. - """ - - if isinstance(index, (int, float)) and index < 0: - index = None - - for i, element in tuple(enumerate(element_seq))[index+1 if index is not None else index:]: - if predicate(element): - return i - return float('-inf') - - -def find_previous(element_seq, predicate, index=None): - """ - Finds and returns the index of the previous element fulfilling the specified predicate preceding to the specified - index, or Infinity. - """ - if isinstance(index, (int, float)) and index >= len(element_seq): - index = None - - for i, element in reversed(tuple(enumerate(element_seq))[:index]): - if predicate(element): - return i - return float('inf') diff --git a/pipenv/patched/prettytoml/elements/traversal/predicates.py b/pipenv/patched/prettytoml/elements/traversal/predicates.py deleted file mode 100644 index f18616bf..00000000 --- a/pipenv/patched/prettytoml/elements/traversal/predicates.py +++ /dev/null @@ -1,48 +0,0 @@ - -""" - The following predicates can be used in the traversal functions directly. -""" - -from ..atomic import AtomicElement -from ..metadata import PunctuationElement, CommentElement, NewlineElement, WhitespaceElement -from prettytoml import tokens -from .. import common - - -atomic = lambda e: isinstance(e, AtomicElement) - - -op_assignment = lambda e: isinstance(e, PunctuationElement) and e.token.type == tokens.TYPE_OP_ASSIGNMENT - - -op_comma = lambda e: isinstance(e, PunctuationElement) and e.token.type == tokens.TYPE_OP_COMMA - - -comment = lambda e: isinstance(e, CommentElement) - - -newline = lambda e: isinstance(e, NewlineElement) - - -non_metadata = lambda e: e.type != common.TYPE_METADATA - - -closing_square_bracket = \ - lambda e: isinstance(e, PunctuationElement) and e.token.type == tokens.TYPE_OP_SQUARE_RIGHT_BRACKET - - -opening_square_bracket = \ - lambda e: isinstance(e, PunctuationElement) and e.token.type == tokens.TYPE_OP_SQUARE_LEFT_BRACKET - - -def table(e): - from ..table import TableElement - return isinstance(e, TableElement) - - -def table_header(e): - from prettytoml.elements.tableheader import TableHeaderElement - return isinstance(e, TableHeaderElement) - - -whitespace = lambda e: isinstance(e, WhitespaceElement) diff --git a/pipenv/patched/prettytoml/errors.py b/pipenv/patched/prettytoml/errors.py deleted file mode 100644 index 23e69eb5..00000000 --- a/pipenv/patched/prettytoml/errors.py +++ /dev/null @@ -1,32 +0,0 @@ - - -class TOMLError(Exception): - """ - All errors raised by this module are descendants of this type. - """ - - -class InvalidTOMLFileError(TOMLError): - pass - - -class NoArrayFoundError(TOMLError): - """ - An array of tables was requested but none exist by the given name. - """ - - -class InvalidValueError(TOMLError): - pass - - -class DuplicateKeysError(TOMLError): - """ - Duplicate keys detected in the parsed file. - """ - - -class DuplicateTablesError(TOMLError): - """ - Duplicate tables detected in the parsed file. - """ diff --git a/pipenv/patched/prettytoml/lexer/__init__.py b/pipenv/patched/prettytoml/lexer/__init__.py deleted file mode 100644 index da32963a..00000000 --- a/pipenv/patched/prettytoml/lexer/__init__.py +++ /dev/null @@ -1,123 +0,0 @@ - -""" -A regular expression based Lexer/tokenizer for TOML. -""" - -from collections import namedtuple -import re -from prettytoml import tokens -from prettytoml.errors import TOMLError - -TokenSpec = namedtuple('TokenSpec', ('type', 're')) - -# Specs of all the valid tokens -_LEXICAL_SPECS = ( - TokenSpec(tokens.TYPE_COMMENT, re.compile(r'^(#.*)\n')), - TokenSpec(tokens.TYPE_STRING, re.compile(r'^("(([^"]|\\")+?[^\\]|([^"]|\\")|)")')), # Single line only - TokenSpec(tokens.TYPE_MULTILINE_STRING, re.compile(r'^(""".*?""")', re.DOTALL)), - TokenSpec(tokens.TYPE_LITERAL_STRING, re.compile(r"^('.*?')")), - TokenSpec(tokens.TYPE_MULTILINE_LITERAL_STRING, re.compile(r"^('''.*?''')", re.DOTALL)), - TokenSpec(tokens.TYPE_BARE_STRING, re.compile(r'^([A-Za-z0-9_-]+)')), - TokenSpec(tokens.TYPE_DATE, re.compile( - r'^([0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]*)?)?(([zZ])|((\+|-)[0-9]{2}:[0-9]{2}))?)')), - TokenSpec(tokens.TYPE_WHITESPACE, re.compile(r'^( |\t)', re.DOTALL)), - TokenSpec(tokens.TYPE_INTEGER, re.compile(r'^(((\+|-)[0-9_]+)|([0-9][0-9_]*))')), - TokenSpec(tokens.TYPE_FLOAT, - re.compile(r'^((((\+|-)[0-9_]+)|([1-9][0-9_]*))(\.[0-9_]+)?([eE](\+|-)?[0-9_]+)?)')), - TokenSpec(tokens.TYPE_BOOLEAN, re.compile(r'^(true|false)')), - TokenSpec(tokens.TYPE_OP_SQUARE_LEFT_BRACKET, re.compile(r'^(\[)')), - TokenSpec(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET, re.compile(r'^(\])')), - TokenSpec(tokens.TYPE_OP_CURLY_LEFT_BRACKET, re.compile(r'^(\{)')), - TokenSpec(tokens.TYPE_OP_CURLY_RIGHT_BRACKET, re.compile(r'^(\})')), - TokenSpec(tokens.TYPE_OP_ASSIGNMENT, re.compile(r'^(=)')), - TokenSpec(tokens.TYPE_OP_COMMA, re.compile(r'^(,)')), - TokenSpec(tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET, re.compile(r'^(\[\[)')), - TokenSpec(tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET, re.compile(r'^(\]\])')), - TokenSpec(tokens.TYPE_OPT_DOT, re.compile(r'^(\.)')), - TokenSpec(tokens.TYPE_NEWLINE, re.compile('^(\n|\r\n)')), -) - - -def _next_token_candidates(source): - matches = [] - for token_spec in _LEXICAL_SPECS: - match = token_spec.re.search(source) - if match: - matches.append(tokens.Token(token_spec.type, match.group(1))) - return matches - - -def _choose_from_next_token_candidates(candidates): - - if len(candidates) == 1: - return candidates[0] - elif len(candidates) > 1: - # Return the maximal-munch with ties broken by natural order of token type. - maximal_munch_length = max(len(token.source_substring) for token in candidates) - maximal_munches = [token for token in candidates if len(token.source_substring) == maximal_munch_length] - return sorted(maximal_munches)[0] # Return the first in sorting by priority - - -def _munch_a_token(source): - """ - Munches a single Token instance if it could recognize one at the beginning of the - given source text, or None if no token type could be recognized. - """ - candidates = _next_token_candidates(source) - return _choose_from_next_token_candidates(candidates) - - -class LexerError(TOMLError): - - def __init__(self, message): - self._message = message - - def __repr__(self): - return self._message - - def __str__(self): - return self._message - - -def tokenize(source, is_top_level=False): - """ - Tokenizes the input TOML source into a stream of tokens. - - If is_top_level is set to True, will make sure that the input source has a trailing newline character - before it is tokenized. - - Raises a LexerError when it fails recognize another token while not at the end of the source. - """ - - # Newlines are going to be normalized to UNIX newlines. - source = source.replace('\r\n', '\n') - - if is_top_level and source and source[-1] != '\n': - source += '\n' - - next_row = 1 - next_col = 1 - next_index = 0 - - while next_index < len(source): - - new_token = _munch_a_token(source[next_index:]) - - if not new_token: - raise LexerError("failed to read the next token at ({}, {}): {}".format( - next_row, next_col, source[next_index:])) - - # Set the col and row on the new token - new_token = tokens.Token(new_token.type, new_token.source_substring, next_col, next_row) - - # Advance the index, row and col count - next_index += len(new_token.source_substring) - for c in new_token.source_substring: - if c == '\n': - next_row += 1 - next_col = 1 - else: - next_col += 1 - - yield new_token - diff --git a/pipenv/patched/prettytoml/lexer/test_lexer.py b/pipenv/patched/prettytoml/lexer/test_lexer.py deleted file mode 100644 index df10b46d..00000000 --- a/pipenv/patched/prettytoml/lexer/test_lexer.py +++ /dev/null @@ -1,153 +0,0 @@ -# -*- coding: utf-8 -*- - -from prettytoml.lexer import _munch_a_token -from prettytoml.lexer import * - -# A mapping from token types to a sequence of pairs of (source_text, expected_matched_text) -valid_tokens = { - tokens.TYPE_COMMENT: ( - ( - '# My very insightful comment about the state of the universe\n# And now for something completely different!', - '# My very insightful comment about the state of the universe', - ), - ), - tokens.TYPE_STRING: ( - ('"a valid hug3 text" "some other string" = 42', '"a valid hug3 text"'), - ( - r'"I\'m a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." "some other string" = 42', - r'"I\'m a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."' - ), - ('"ʎǝʞ" key', '"ʎǝʞ"'), - ('""', '""'), - ('"t"', '"t"'), - ), - tokens.TYPE_MULTILINE_STRING: ( - ('"""\nRoses are red\nViolets are blue""" """other text"""', '"""\nRoses are red\nViolets are blue"""'), - ), - tokens.TYPE_LITERAL_STRING: ( - (r"'This is \ \n a \\ literal string' 'another \ literal string'", r"'This is \ \n a \\ literal string'"), - ), - tokens.TYPE_MULTILINE_LITERAL_STRING: ( - ( - "'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n''' '''some other\n\n\t string'''", - "'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''" - ), - ), - tokens.TYPE_DATE: ( - ('1979-05-27 5345', '1979-05-27'), - ('1979-05-27T07:32:00Z something', '1979-05-27T07:32:00Z'), - ('1979-05-27T00:32:00-07:00 ommm', '1979-05-27T00:32:00-07:00'), - ('1979-05-27T00:32:00.999999-07:00 2346', '1979-05-27T00:32:00.999999-07:00'), - ), - tokens.TYPE_WHITESPACE: ( - (' \t\n \r some_text', ' '), - ), - tokens.TYPE_INTEGER: ( - ('+99 "number"', "+99"), - ('42 fwfwef', "42"), - ('-17 fh34g34g', "-17"), - ('5_349_221 apples', "5_349_221"), - ('-1_2_3_4_5 steps', '-1_2_3_4_5') - ), - tokens.TYPE_FLOAT: ( - ('1.0 fwef', '1.0'), - ('3.1415 g4g', '3.1415'), - ('-0.01 433re', '-0.01'), - ('5e+2_2 ersdvf', '5e+2_2'), - ('1e6 ewe23', '1e6'), - ('-2E-2.2 3 rf23', '-2E-2'), - ('6.626e-34 +234f', '6.626e-34'), - ('9_224_617.445_991_228_313 f1ewer 23f4h = nonesense', '9_224_617.445_991_228_313'), - ('1e1_000 2346f,ef2!!', '1e1_000'), - ), - tokens.TYPE_BOOLEAN: ( - ('false business = true', 'false'), - ('true true', 'true'), - ), - tokens.TYPE_OP_SQUARE_LEFT_BRACKET: ( - ('[table_name]', '['), - ), - tokens.TYPE_OP_SQUARE_RIGHT_BRACKET: ( - (']\nbusiness = awesome', ']'), - ), - tokens.TYPE_OP_CURLY_LEFT_BRACKET: ( - ('{item_exists = no}', '{'), - ), - tokens.TYPE_OP_CURLY_RIGHT_BRACKET: ( - ('} moving on', '}'), - ), - tokens.TYPE_OP_COMMA: ( - (',item2,item4', ','), - ), - tokens.TYPE_OP_ASSIGNMENT: ( - ('== 42', '='), - ), - tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET: ( - ('[[array.of.tables]]', '[['), - ), - tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET: ( - (']] item=3', ']]'), - ), - tokens.TYPE_BARE_STRING: ( - ('key another', 'key'), - ('bare_key 2fews', 'bare_key'), - ('bare-key kfcw', 'bare-key'), - ), - tokens.TYPE_OPT_DOT: ( - ('."another key"', '.'), - ('.subname', '.'), - ), - tokens.TYPE_NEWLINE: ( - ('\n\r \n', '\n'), - ) -} - -# A mapping from a token type to a sequence of (source, matched_text) pairs that shouldn't result from consuming the -# source text. -invalid_tokens = { - tokens.TYPE_INTEGER: ( - ('_234_423', ''), - ('0446234234', ''), - ), - tokens.TYPE_STRING: ( - ('"""', '"""'), - ), - tokens.TYPE_BOOLEAN: ( - ('True', 'True'), - ('True', 'true'), - ), - tokens.TYPE_FLOAT: ( - ('', ''), - ) -} - - -def test_valid_tokenizing(): - for token_type in valid_tokens: - for (source, expected_match) in valid_tokens[token_type]: - - token = _munch_a_token(source) - assert token, "Failed to tokenize: {}\nExpected: {}\nOut of: {}\nGot nothing!".format( - token_type, expected_match, source) - - assert token.type == token_type, \ - "Expected type: {}\nOut of: {}\nThat matched: {}\nOf type: {}".format( - token_type, source, token.source_substring, token.type) - assert token.source_substring == expected_match - - -def test_invalid_tokenizing(): - for token_type in invalid_tokens: - for source, expected_match in invalid_tokens[token_type]: - token = _munch_a_token(source) - if token: - assert not (token.type == token_type and token.source_substring == expected_match) - - -def test_token_type_order(): - type_a = tokens.TokenType('a', 5, is_metadata=False) - type_b = tokens.TokenType('b', 0, is_metadata=False) - type_c = tokens.TokenType('c', 3, is_metadata=False) - - assert type_b < type_c < type_a - assert type_a > type_c > type_b diff --git a/pipenv/patched/prettytoml/parser/__init__.py b/pipenv/patched/prettytoml/parser/__init__.py deleted file mode 100644 index 4cf600c0..00000000 --- a/pipenv/patched/prettytoml/parser/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ - -""" - A parser for TOML tokens into TOML elements. -""" - - -from prettytoml.parser.errors import ParsingError - - -def parse_tokens(tokens): - """ - Parses the given token sequence into a sequence of top-level TOML elements. - - Raises ParserError on invalid TOML input. - """ - from .tokenstream import TokenStream - return _parse_token_stream(TokenStream(tokens)) - - -def _parse_token_stream(token_stream): - """ - Parses the given token_stream into a sequence of top-level TOML elements. - - Raises ParserError on invalid input TOML. - """ - from .parser import toml_file_elements - from .elementsanitizer import sanitize - - elements, pending = toml_file_elements(token_stream) - - if not pending.at_end: - raise ParsingError('Failed to parse line {}'.format(pending.head.row)) - - return sanitize(elements) diff --git a/pipenv/patched/prettytoml/parser/elementsanitizer.py b/pipenv/patched/prettytoml/parser/elementsanitizer.py deleted file mode 100644 index bec4893a..00000000 --- a/pipenv/patched/prettytoml/parser/elementsanitizer.py +++ /dev/null @@ -1,58 +0,0 @@ -from prettytoml import elements -from prettytoml.elements.table import TableElement -from prettytoml.elements.tableheader import TableHeaderElement -from prettytoml.errors import InvalidTOMLFileError -from prettytoml.util import PeekableIterator - - -def sanitize(_elements): - """ - Finds TableHeader elements that are not followed by TableBody elements and inserts empty TableElement - right after those. - """ - - output = list(_elements) - - def find_next_table_header(after=-1): - return next((i for (i, element) in enumerate(output) - if i > after and isinstance(element, TableHeaderElement)), float('-inf')) - - def find_next_table_body(after=-1): - return next((i for (i, element) in enumerate(output) - if i > after and isinstance(element, TableElement)), float('-inf')) - - next_table_header_i = find_next_table_header() - while next_table_header_i >= 0: - - following_table_header_i = find_next_table_header(next_table_header_i) - following_table_body_i = find_next_table_body(next_table_header_i) - - if (following_table_body_i < 0) or \ - (following_table_header_i >= 0 and (following_table_header_i < following_table_body_i)): - output.insert(next_table_header_i+1, TableElement(tuple())) - - next_table_header_i = find_next_table_header(next_table_header_i) - - return output - - -def validate_sanitized(_elements): - - # Non-metadata elements must start with an optional TableElement, followed by - # zero or more (TableHeaderElement, TableElement) pairs. - - if not _elements: - return - - it = PeekableIterator(e for e in _elements if e.type != elements.TYPE_METADATA) - - if isinstance(it.peek(), TableElement): - it.next() - - while it.peek(): - if not isinstance(it.peek(), TableHeaderElement): - raise InvalidTOMLFileError - it.next() - if not isinstance(it.peek(), TableElement): - raise InvalidTOMLFileError - it.next() diff --git a/pipenv/patched/prettytoml/parser/errors.py b/pipenv/patched/prettytoml/parser/errors.py deleted file mode 100644 index eca12f32..00000000 --- a/pipenv/patched/prettytoml/parser/errors.py +++ /dev/null @@ -1,17 +0,0 @@ -from prettytoml.errors import TOMLError - - -class ParsingError(TOMLError): - - def __init__(self, message='', token=None): - self.message = message - self.token = token - - def __repr__(self): - if self.message and self.token: - return "{} at row {} and col {}".format(self.message, self.token.row, self.token.col) - else: - return self.message - - def __str__(self): - return repr(self) diff --git a/pipenv/patched/prettytoml/parser/parser.py b/pipenv/patched/prettytoml/parser/parser.py deleted file mode 100644 index e61c0db5..00000000 --- a/pipenv/patched/prettytoml/parser/parser.py +++ /dev/null @@ -1,376 +0,0 @@ - -""" - A Recursive Descent implementation of a lexical parser for TOML. - - Grammar: - -------- - - Newline -> NEWLINE - Comment -> COMMENT Newline - LineTerminator -> Comment | Newline - Space -> WHITESPACE Space | WHITESPACE | EMPTY - TableHeader -> Space [ Space TableHeaderName Space ] Space LineTerminator | - Space [[ Space TableHeaderName Space ]] Space LineTerminator - TableHeaderName -> STRING Space '.' Space TableHeaderName | STRING - Atomic -> STRING | INTEGER | FLOAT | DATE | BOOLEAN - - Array -> '[' Space ArrayInternal Space ']' | '[' Space ArrayInternal Space LineTerminator Space ']' - ArrayInternal -> LineTerminator Space ArrayInternal | Value Space ',' Space LineTerminator Space ArrayInternal | - Value Space ',' Space ArrayInternal | LineTerminator | Value | EMPTY - - InlineTable -> '{' Space InlineTableInternal Space '}' - InlineTableKeyValuePair = STRING Space '=' Space Value - InlineTableInternal -> InlineTableKeyValuePair Space ',' Space InlineTableInternal | - InlineTableKeyValuePair | Empty - - Value -> Atomic | InlineTable | Array - KeyValuePair -> Space STRING Space '=' Space Value Space LineTerminator - - TableBody -> KeyValuePair TableBody | EmptyLine TableBody | EmptyLine | KeyValuePair - - EmptyLine -> Space LineTerminator - FileEntry -> TableHeader | TableBody - - TOMLFileElements -> FileEntry TOMLFileElements | FileEntry | EmptyLine | EMPTY -""" - -from prettytoml import tokens -from prettytoml.elements.array import ArrayElement -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.inlinetable import InlineTableElement -from prettytoml.elements.metadata import NewlineElement, CommentElement, WhitespaceElement, PunctuationElement -from prettytoml.elements.table import TableElement -from prettytoml.elements.tableheader import TableHeaderElement - -from prettytoml.parser.recdesc import capture_from -from prettytoml.parser.errors import ParsingError -from prettytoml.parser.tokenstream import TokenStream - -""" - Non-terminals are represented as functions which return (RESULT, pending_token_stream), or raise ParsingError. -""" - - -def token(token_type): - def factory(ts): - t = ts.head - if t.type != token_type: - raise ParsingError('Expected a token of type {}'.format(token_type)) - return t, ts.tail - return factory - - -def newline_element(token_stream): - """ - Returns NewlineElement, pending_token_stream or raises ParsingError. - """ - captured = capture_from(token_stream).find(token(tokens.TYPE_NEWLINE)) - return NewlineElement(captured.value()), captured.pending_tokens - - -def comment_tokens(ts1): - c1 = capture_from(ts1).find(token(tokens.TYPE_COMMENT)).and_find(token(tokens.TYPE_NEWLINE)) - return c1.value(), c1.pending_tokens - - -def comment_element(token_stream): - """ - Returns CommentElement, pending_token_stream or raises ParsingError. - """ - captured = capture_from(token_stream).find(comment_tokens) - return CommentElement(captured.value()), captured.pending_tokens - - -def line_terminator_tokens(token_stream): - captured = capture_from(token_stream).find(comment_tokens).or_find(token(tokens.TYPE_NEWLINE)) - return captured.value(), captured.pending_tokens - - -def line_terminator_element(token_stream): - captured = capture_from(token_stream).find(comment_element).or_find(newline_element) - return captured.value('Expected a comment or a newline')[0], captured.pending_tokens - - -def zero_or_more_tokens(token_type): - - def factory(token_stream): - def more(ts): - c = capture_from(ts).find(token(token_type)).and_find(zero_or_more_tokens(token_type)) - return c.value(), c.pending_tokens - - def two(ts): - c = capture_from(ts).find(token(tokens.TYPE_WHITESPACE)) - return c.value(), c.pending - - def zero(ts): - return tuple(), ts - - captured = capture_from(token_stream).find(more).or_find(two).or_find(zero) - return captured.value(), captured.pending_tokens - - return factory - - -def space_element(token_stream): - captured = capture_from(token_stream).find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)) - return WhitespaceElement([t for t in captured.value() if t]), captured.pending_tokens - - -def string_token(token_stream): - captured = capture_from(token_stream).\ - find(token(tokens.TYPE_BARE_STRING)).\ - or_find(token(tokens.TYPE_STRING)).\ - or_find(token(tokens.TYPE_LITERAL_STRING)).\ - or_find(token(tokens.TYPE_MULTILINE_STRING)).\ - or_find(token(tokens.TYPE_MULTILINE_LITERAL_STRING)) - return captured.value('Expected a string'), captured.pending_tokens - - -def string_element(token_stream): - captured = capture_from(token_stream).find(string_token) - return AtomicElement(captured.value()), captured.pending_tokens - - -def table_header_name_tokens(token_stream): - - def one(ts): - c = capture_from(ts).\ - find(string_token).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(token(tokens.TYPE_OPT_DOT)).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(table_header_name_tokens) - return c.value(), c.pending_tokens - - captured = capture_from(token_stream).find(one).or_find(string_token) - return captured.value(), captured.pending_tokens - - -def table_header_element(token_stream): - - def single(ts1): - c1 = capture_from(ts1).\ - find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(token(tokens.TYPE_OP_SQUARE_LEFT_BRACKET)).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(table_header_name_tokens).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(token(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET)).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(line_terminator_tokens) - - return c1.value(), c1.pending_tokens - - def double(ts2): - c2 = capture_from(ts2).\ - find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(token(tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET)).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(table_header_name_tokens).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(token(tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET)).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(line_terminator_tokens) - - return c2.value(), c2.pending_tokens - - captured = capture_from(token_stream).find(single).or_find(double) - return TableHeaderElement(captured.value()), captured.pending_tokens - - -def atomic_element(token_stream): - captured = capture_from(token_stream).\ - find(string_token).\ - or_find(token(tokens.TYPE_INTEGER)).\ - or_find(token(tokens.TYPE_FLOAT)).\ - or_find(token(tokens.TYPE_DATE)).\ - or_find(token(tokens.TYPE_BOOLEAN)) - return AtomicElement(captured.value('Expected an atomic primitive value')), captured.pending_tokens - - -def punctuation_element(token_type): - def factory(ts): - c = capture_from(ts).find(token(token_type)) - return PunctuationElement(c.value('Expected the punctuation element: {}'.format(token_type))), c.pending_tokens - return factory - - -def value(token_stream): - captured = capture_from(token_stream).\ - find(atomic_element).\ - or_find(array_element).\ - or_find(inline_table_element) - return captured.value('Expected a primitive value, array or an inline table'), captured.pending_tokens - - -def array_internal(ts): - - def zero(ts0): - c = capture_from(ts0).\ - and_find(line_terminator_element).\ - and_find(space_element).\ - and_find(array_internal) - return c.value(), c.pending_tokens - - def one(ts1): - c = capture_from(ts1).\ - find(value).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_COMMA)).\ - and_find(space_element).\ - and_find(line_terminator_element).\ - and_find(space_element).\ - and_find(array_internal) - return c.value(), c.pending_tokens - - def two(ts2): - c = capture_from(ts2).\ - find(value).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_COMMA)).\ - and_find(space_element).\ - and_find(array_internal) - return c.value(), c.pending_tokens - - def three(ts3): - c = capture_from(ts3).\ - find(space_element).\ - and_find(line_terminator_element) - return c.value(), c.pending_tokens - - captured = capture_from(ts).find(zero).or_find(one).or_find(two).or_find(three).or_find(value).or_empty() - return captured.value(), captured.pending_tokens - - -def array_element(token_stream): - - def one(ts1): - ca = capture_from(ts1).\ - find(punctuation_element(tokens.TYPE_OP_SQUARE_LEFT_BRACKET)).\ - and_find(space_element).\ - and_find(array_internal).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET)) - return ca.value(), ca.pending_tokens - - def two(ts2): - ca = capture_from(ts2).\ - find(punctuation_element(tokens.TYPE_OP_SQUARE_LEFT_BRACKET)).\ - and_find(space_element).\ - and_find(array_internal).\ - and_find(space_element).\ - and_find(line_terminator_element).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET)) - return ca.value(), ca.pending_tokens - - captured = capture_from(token_stream).find(one).or_find(two) - return ArrayElement(captured.value()), captured.pending_tokens - - -def inline_table_element(token_stream): - - # InlineTableElement -> '{' Space InlineTableInternal Space '}' - # InlineTableKeyValuePair = STRING Space '=' Space Value - # InlineTableInternal -> InlineTableKeyValuePair Space ',' Space InlineTableInternal | - # InlineTableKeyValuePair | Empty - - def key_value(ts): - ca = capture_from(ts).\ - find(string_element).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_ASSIGNMENT)).\ - and_find(space_element).\ - and_find(value) - return ca.value(), ca.pending_tokens - - def internal(ts): - def one(ts1): - c1 = capture_from(ts1).\ - find(key_value).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_COMMA)).\ - and_find(space_element).\ - and_find(internal) - return c1.value(), c1.pending_tokens - - c = capture_from(ts).find(one).or_find(key_value).or_empty() - return c.value(), c.pending_tokens - - captured = capture_from(token_stream).\ - find(punctuation_element(tokens.TYPE_OP_CURLY_LEFT_BRACKET)).\ - and_find(space_element).\ - and_find(internal).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_CURLY_RIGHT_BRACKET)) - - return InlineTableElement(captured.value()), captured.pending_tokens - - -def key_value_pair(token_stream): - captured = capture_from(token_stream).\ - find(space_element).\ - and_find(string_element).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_ASSIGNMENT)).\ - and_find(space_element).\ - and_find(value).\ - and_find(space_element).\ - and_find(line_terminator_element) - return captured.value(), captured.pending_tokens - - -def table_body_elements(token_stream): - - # TableBody -> KeyValuePair TableBody | EmptyLine TableBody | EmptyLine | KeyValuePair - - def one(ts1): - c = capture_from(ts1).\ - find(key_value_pair).\ - and_find(table_body_elements) - return c.value(), c.pending_tokens - - def two(ts2): - c = capture_from(ts2).\ - find(empty_line_elements).\ - and_find(table_body_elements) - return c.value(), c.pending_tokens - - captured = capture_from(token_stream).\ - find(one).\ - or_find(two).\ - or_find(empty_line_elements).\ - or_find(key_value_pair) - - return captured.value(), captured.pending_tokens - - -def table_body_element(token_stream): - captured = capture_from(token_stream).find(table_body_elements) - return TableElement(captured.value()), captured.pending_tokens - - -def empty_line_tokens(ts1): - c1 = capture_from(ts1).find(space_element).and_find(line_terminator_element) - return c1.value(), c1.pending_tokens - - -def empty_line_elements(token_stream): - captured = capture_from(token_stream).find(empty_line_tokens) - return captured.value(), captured.pending_tokens - - -def file_entry_element(token_stream): - captured = capture_from(token_stream).find(table_header_element).\ - or_find(table_body_element) - return captured.value(), captured.pending_tokens - - -def toml_file_elements(token_stream): - - def one(ts1): - c1 = capture_from(ts1).find(file_entry_element).and_find(toml_file_elements) - return c1.value(), c1.pending_tokens - - captured = capture_from(token_stream).find(one).or_find(file_entry_element).or_empty() - return captured.value(), captured.pending_tokens diff --git a/pipenv/patched/prettytoml/parser/recdesc.py b/pipenv/patched/prettytoml/parser/recdesc.py deleted file mode 100644 index 8731dba3..00000000 --- a/pipenv/patched/prettytoml/parser/recdesc.py +++ /dev/null @@ -1,114 +0,0 @@ -from prettytoml.parser.errors import ParsingError -from prettytoml.parser.tokenstream import TokenStream - - -class Capturer: - """ - Recursive-descent matching DSL. Yeah.. - """ - - def __init__(self, token_stream, value=tuple(), dormant_error=None): - self._token_stream = token_stream - self._value = value - self._dormant_error = dormant_error - - def find(self, finder): - """ - Searches the token stream using the given finder. - - `finder(ts)` is a function that accepts a `TokenStream` instance and returns `(element, pending_ts)` - where `element` is the found "something" or a sequence of "somethings", and `pending_ts` the unconsumed - `TokenStream`. - - `finder(ts)` can raise `ParsingError` to indicate that it couldn't find anything, or - a `TokenStream.EndOfStream` to indicate a premature end of the TokenStream. - - This method returns a Capturer instance that can be further used to find more and more "somethings". The value - at any given moment can be retrieved via the `Capturer.value()` method. - """ - - try: - - # Execute finder! - element, pending_ts = finder(self._token_stream) - - # If result is not a sequence, make it so - if not isinstance(element, (tuple, list)): - element = (element,) - - # Return a Capturer with accumulated findings - return Capturer(pending_ts, value=self.value() + element) - - except ParsingError as e: - - # Failed to find, store error in returned value - return Capturer(self._token_stream, dormant_error=e) - - except TokenStream.EndOfStream as e: - - # Premature end of stream, store error in returned value - return Capturer(self._token_stream, dormant_error=e) - - def value(self, parsing_expectation_msg=None): - """ - Returns the accumulated values found as a sequence of values, or raises an encountered dormant error. - - If parsing_expectation_msg is specified and a dormant_error is a ParsingError, the expectation message is used - instead in it. - """ - - if self._dormant_error: - if parsing_expectation_msg and isinstance(self._dormant_error, ParsingError): - raise ParsingError(parsing_expectation_msg, token=self._token_stream.head) - else: - raise self._dormant_error - return self._value - - @property - def pending_tokens(self): - """ - Returns a TokenStream with the pending tokens yet to be processed. - """ - return self._token_stream - - def or_find(self, finder): - """ - If a dormant_error is present, try this new finder instead. If not, does nothing. - """ - if self._dormant_error: - return Capturer(self._token_stream).find(finder) - else: - return self - - def or_end_of_file(self): - """ - Discards any errors if at end of the stream. - """ - if isinstance(self._dormant_error, TokenStream.EndOfStream): - return Capturer(self.pending_tokens, value=self._value) - else: - return self - - def or_empty(self): - """ - Discards any previously-encountered dormant error. - """ - if self._dormant_error: - return Capturer(self.pending_tokens, value=self._value) - else: - return self - - def and_find(self, finder): - """ - Accumulate new "somethings" to the stored value using the given finder. - """ - - if self._dormant_error: - return Capturer(self.pending_tokens, dormant_error=self._dormant_error) - - return Capturer(self.pending_tokens, self.value()).find(finder) - - -def capture_from(token_stream): - return Capturer(token_stream) - diff --git a/pipenv/patched/prettytoml/parser/test_parser.py b/pipenv/patched/prettytoml/parser/test_parser.py deleted file mode 100644 index 40dd3dba..00000000 --- a/pipenv/patched/prettytoml/parser/test_parser.py +++ /dev/null @@ -1,156 +0,0 @@ -from prettytoml.elements.array import ArrayElement -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import CommentElement, NewlineElement, WhitespaceElement -from prettytoml.elements.tableheader import TableHeaderElement -from prettytoml.lexer import tokenize -from prettytoml.parser import parser -from prettytoml.parser.tokenstream import TokenStream - - -def test_line_terminator_1(): - tokens = tokenize('# Sup\n') - ts = TokenStream(tokens) - element, pending_ts = parser.line_terminator_element(ts) - - assert isinstance(element, CommentElement) - assert pending_ts.offset == 2 - assert ts.offset == 0 - - -def test_line_terminator_2(): - tokens = tokenize('\n') - ts = TokenStream(tokens) - element, pending_ts = parser.line_terminator_element(ts) - - assert isinstance(element, NewlineElement) - assert pending_ts.offset == 1 - assert ts.offset == 0 - - -def test_space_1(): - ts = TokenStream(tokenize(' noo')) - space_element, pending_ts = parser.space_element(ts) - - assert isinstance(space_element, WhitespaceElement) - assert len(space_element.tokens) == 2 - assert pending_ts.offset == 2 - assert ts.offset == 0 - - -def test_space_2(): - ts = TokenStream(tokenize(' noo')) - space_element, pending_ts = parser.space_element(ts) - - assert isinstance(space_element, WhitespaceElement) - assert len(space_element.tokens) == 1 - assert pending_ts.offset == 1 - assert ts.offset == 0 - - -def test_space_3(): - ts = TokenStream(tokenize('noo')) - space_element, pending_ts = parser.space_element(ts) - - assert isinstance(space_element, WhitespaceElement) - assert len(space_element.tokens) == 0 - assert pending_ts.offset == 0 - assert ts.offset == 0 - - -def test_table_header(): - ts = TokenStream(tokenize(" [ namez . namey . namex ] \n other things")) - table_header_element, pending_tokens = parser.table_header_element(ts) - - assert isinstance(table_header_element, TableHeaderElement) - assert len(pending_tokens) == 4 - - -def test_atomic_element(): - e1, p1 = parser.atomic_element(TokenStream(tokenize('42 not'))) - assert isinstance(e1, AtomicElement) and e1.value == 42 - assert len(p1) == 2 - - e2, p2 = parser.atomic_element(TokenStream(tokenize('not 42'))) - assert isinstance(e2, AtomicElement) and e2.value == 'not' - assert len(p2) == 2 - - -def test_array(): - array_element, pending_ts = parser.array_element(TokenStream(tokenize('[ 3, 4, 5,6,7] '))) - - assert isinstance(array_element, ArrayElement) - assert len(array_element) == 5 - assert len(pending_ts) == 1 - - -def test_array_2(): - - text = """[ - "alpha", - "omega" -]""" - - array_element, pending_ts = parser.array_element(TokenStream(tokenize(text))) - - assert array_element[0] == 'alpha' - assert array_element[1] == 'omega' - - -def test_empty_array(): - - text = '[]' - - array_element, pending_ts = parser.array_element(TokenStream(tokenize(text))) - - assert isinstance(array_element, ArrayElement) - assert pending_ts.at_end - - -def test_inline_table(): - inline_table, pending_ts = parser.inline_table_element(TokenStream(tokenize('{ "id"= 42,test = name} vroom'))) - - assert set(inline_table.keys()) == {'id', 'test'} - assert len(pending_ts) == 2 - assert inline_table['id'] == 42 - assert inline_table['test'] == 'name' - - -def test_table_body(): - table_body, pending_ts = parser.table_body_element(TokenStream(tokenize(' name= "test" # No way man!\nid =42\n vvv'))) - assert set(table_body.keys()) == {'name', 'id'} - assert len(pending_ts) == 2 - assert table_body['name'] == 'test' - assert table_body['id'] == 42 - - -def test_key_value_pair(): - text = """hosts = [ - "alpha", - "omega" -] -""" - - parsed, pending_ts = parser.key_value_pair(TokenStream(tokenize(text))) - - assert isinstance(parsed[1], AtomicElement) - assert isinstance(parsed[5], ArrayElement) - - -def test_table_body_2(): - - text = """ -data = [ ["gamma", "delta"], [1, 2] ] - -# Line breaks are OK when inside arrays -hosts = [ - "alpha", - "omega" -] - -str_multiline = wohoo -""" - - table_body, pending_ts = parser.table_body_element(TokenStream(tokenize(text))) - - assert len(pending_ts) == 0 - diff --git a/pipenv/patched/prettytoml/parser/tokenstream.py b/pipenv/patched/prettytoml/parser/tokenstream.py deleted file mode 100644 index 2a2fdc25..00000000 --- a/pipenv/patched/prettytoml/parser/tokenstream.py +++ /dev/null @@ -1,39 +0,0 @@ - -class TokenStream: - """ - An immutable subset of a token sequence - """ - - class EndOfStream(Exception): - pass - - Nothing = tuple() - - def __init__(self, _tokens, offset=0): - if isinstance(_tokens, tuple): - self._tokens = _tokens - else: - self._tokens = tuple(_tokens) - self._head_index = offset - - def __len__(self): - return len(self._tokens) - self.offset - - @property - def head(self): - try: - return self._tokens[self._head_index] - except IndexError: - raise TokenStream.EndOfStream - - @property - def tail(self): - return TokenStream(self._tokens, offset=self._head_index+1) - - @property - def offset(self): - return self._head_index - - @property - def at_end(self): - return self.offset >= len(self._tokens) diff --git a/pipenv/patched/prettytoml/prettifier/__init__.py b/pipenv/patched/prettytoml/prettifier/__init__.py deleted file mode 100644 index 97ac1619..00000000 --- a/pipenv/patched/prettytoml/prettifier/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -from . import deindentanonymoustable, tableindent, tableassignment -from prettytoml.prettifier import tablesep, commentspace, linelength, tableentrysort - -""" - TOMLFile prettifiers - - Each prettifier is a function that accepts a sequence of Element instances that make up a - TOML file and it is allowed to modify it as it pleases. -""" - - -UNIFORM_TABLE_INDENTATION = tableindent.table_entries_should_be_uniformly_indented -UNIFORM_TABLE_ASSIGNMENT_SPACING = tableassignment.table_assignment_spacing -ANONYMOUS_TABLE_INDENTATION = deindentanonymoustable.deindent_anonymous_table -COMMENT_SPACING = commentspace.comment_space -TABLE_SPACING = tablesep.table_separation -LINE_LENGTH_ENFORCERS = linelength.line_length_limiter -TABLE_ENTRY_SORTING = tableentrysort.sort_table_entries - - -ALL = ( - TABLE_SPACING, # Must be before COMMENT_SPACING - COMMENT_SPACING, # Must be after TABLE_SPACING - UNIFORM_TABLE_INDENTATION, - UNIFORM_TABLE_ASSIGNMENT_SPACING, - ANONYMOUS_TABLE_INDENTATION, - LINE_LENGTH_ENFORCERS, - TABLE_ENTRY_SORTING, -) - - -def prettify(toml_file_elements, prettifiers=ALL): - """ - Prettifies a sequence of element instances according to pre-defined set of formatting rules. - """ - elements = toml_file_elements[:] - for prettifier in prettifiers: - elements = prettifier(elements) - return elements diff --git a/pipenv/patched/prettytoml/prettifier/commentspace.py b/pipenv/patched/prettytoml/prettifier/commentspace.py deleted file mode 100644 index fd549349..00000000 --- a/pipenv/patched/prettytoml/prettifier/commentspace.py +++ /dev/null @@ -1,35 +0,0 @@ - -from prettytoml.elements import traversal as t, factory as element_factory -from prettytoml.elements.table import TableElement - - -def comment_space(toml_file_elements): - """ - Rule: Line-terminating comments should always be prefixed by a single tab character whitespace only. - """ - elements = toml_file_elements[:] - for element in elements: - if isinstance(element, TableElement): - _do_table(element.sub_elements) - return elements - - -def _do_table(table_elements): - - # Iterator index - i = float('-inf') - - def next_newline(): - return t.find_following(table_elements, t.predicates.newline, i) - - def next_comment(): - return t.find_following(table_elements, t.predicates.comment, i) - - def last_non_metadata(): - return t.find_previous(table_elements, t.predicates.non_metadata, next_comment()) - - while next_comment() >= 0: - if i < last_non_metadata() < next_comment() < next_newline(): - del table_elements[last_non_metadata()+1:next_comment()] - table_elements.insert(next_comment(), element_factory.create_whitespace_element(char='\t', length=1)) - i = next_newline() diff --git a/pipenv/patched/prettytoml/prettifier/common.py b/pipenv/patched/prettytoml/prettifier/common.py deleted file mode 100644 index dd1e01a2..00000000 --- a/pipenv/patched/prettytoml/prettifier/common.py +++ /dev/null @@ -1,54 +0,0 @@ - -from itertools import * -from prettytoml.elements.common import TokenElement -from prettytoml.elements.metadata import NewlineElement - - -def text_to_elements(toml_text): - from ..lexer import tokenize - from ..parser import parse_tokens - return parse_tokens(tokenize(toml_text)) - - -def elements_to_text(toml_elements): - return ''.join(e.serialized() for e in toml_elements) - - -def assert_prettifier_works(source_text, expected_text, prettifier_func): - assert expected_text == elements_to_text(prettifier_func(text_to_elements(source_text))) - - -def lines(elements): - """ - Splits a sequence of elements into a sub-sequence of each line. - - A line is defined as a sequence of elements terminated by a NewlineElement. - """ - - def __next_line(es): - # Returns the next line and the remaining sequence of elements - line = tuple(takewhile(lambda e: not isinstance(e, NewlineElement), es)) - line += (es[len(line)],) - return line, es[len(line):] - - left_elements = tuple(elements) - while left_elements: - line, left_elements = __next_line(left_elements) - yield line - - -def non_empty_elements(elements): - """ - Filters out TokenElement instances with zero tokens. - """ - return filter(lambda e: not (isinstance(e, TokenElement) and not e.tokens), elements) - - -def index(predicate, seq): - """ - Returns the index of the element satisfying the given predicate, or None. - """ - try: - return next(i for (i, e) in enumerate(seq) if predicate(e)) - except StopIteration: - return None diff --git a/pipenv/patched/prettytoml/prettifier/deindentanonymoustable.py b/pipenv/patched/prettytoml/prettifier/deindentanonymoustable.py deleted file mode 100644 index a661f704..00000000 --- a/pipenv/patched/prettytoml/prettifier/deindentanonymoustable.py +++ /dev/null @@ -1,43 +0,0 @@ -import operator -from prettytoml.elements import traversal as t, traversal -from itertools import * -from functools import * -from prettytoml.elements.metadata import WhitespaceElement -from prettytoml.elements.table import TableElement -from prettytoml.prettifier import common - - -def deindent_anonymous_table(toml_file_elements): - """ - Rule: Anonymous table should never be indented. - """ - - anonymous_table_index = _find_anonymous_table(toml_file_elements) - if anonymous_table_index is None: - return toml_file_elements - - return toml_file_elements[:anonymous_table_index] + \ - [_unindent_table(toml_file_elements[anonymous_table_index])] + \ - toml_file_elements[anonymous_table_index+1:] - - -def _unindent_table(table_element): - table_lines = tuple(common.lines(table_element.sub_elements)) - unindented_lines = tuple(tuple(dropwhile(lambda e: isinstance(e, WhitespaceElement), line)) for line in table_lines) - return TableElement(reduce(operator.concat, unindented_lines)) - - -def _find_anonymous_table(toml_file_elements): - """ - Finds and returns the index of the TableElement comprising the anonymous table or None. - """ - - first_table_index = common.index(t.predicates.table, toml_file_elements) - first_table_header_index = common.index(t.predicates.table_header, toml_file_elements) - - if first_table_header_index is None: - return first_table_index - elif first_table_index < first_table_header_index: - return first_table_index - - diff --git a/pipenv/patched/prettytoml/prettifier/linelength.py b/pipenv/patched/prettytoml/prettifier/linelength.py deleted file mode 100644 index 67d3a112..00000000 --- a/pipenv/patched/prettytoml/prettifier/linelength.py +++ /dev/null @@ -1,62 +0,0 @@ -import operator -from prettytoml import tokens -from prettytoml.prettifier import common -from prettytoml.elements import traversal as t, factory as element_factory -from prettytoml.elements.array import ArrayElement -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.inlinetable import InlineTableElement -from prettytoml.elements.table import TableElement -from functools import * - - -MAXIMUM_LINE_LENGTH = 120 - - -def line_length_limiter(toml_file_elements): - """ - Rule: Lines whose lengths exceed 120 characters whose values are strings, arrays should have the array or - string value broken onto multiple lines - """ - return tuple(_fixed_table(e) if isinstance(e, TableElement) else e for e in toml_file_elements) - - -def _fixed_table(table_element): - """ - Returns a new TableElement. - """ - assert isinstance(table_element, TableElement) - lines = tuple(common.lines(table_element.sub_elements)) - fixed_lines = tuple(_fixed_line(l) if _line_length(l) > MAXIMUM_LINE_LENGTH else l for l in lines) - return TableElement(sub_elements=tuple(reduce(operator.concat, fixed_lines))) - - -def _line_length(line_elements): - """ - Returns the character length of the serialized elements of the given line. - """ - return sum(len(e.serialized()) for e in line_elements) - - -def _fixed_line(line_elements): - - def line_value_index(): - # Returns index of value element in the line - key_index = t.find_following(line_elements, t.predicates.non_metadata) - return t.find_following(line_elements, t.predicates.non_metadata, key_index) - - def multiline_equivalent(element): - if isinstance(element, AtomicElement) and tokens.is_string(element.first_token): - return element_factory.create_multiline_string(element.value, MAXIMUM_LINE_LENGTH) - elif isinstance(element, ArrayElement): - element.turn_into_multiline() - return element - else: - return element - - line_elements = tuple(line_elements) - value_index = line_value_index() - if value_index >= 0: - return line_elements[:value_index] + (multiline_equivalent(line_elements[value_index]),) + \ - line_elements[value_index+1:] - else: - return line_elements diff --git a/pipenv/patched/prettytoml/prettifier/tableassignment.py b/pipenv/patched/prettytoml/prettifier/tableassignment.py deleted file mode 100644 index 1d35d698..00000000 --- a/pipenv/patched/prettytoml/prettifier/tableassignment.py +++ /dev/null @@ -1,40 +0,0 @@ - -from prettytoml.elements import traversal as t, factory as element_factory - - -def table_assignment_spacing(toml_file_elements): - """ - Rule: Every key and value pair in any table should be separated the triplet - (single space character, an assignment character =, single space character) - """ - elements = toml_file_elements[:] - for table_element in (e for e in elements if t.predicates.table(e)): - _do_table(table_element) - return elements - - -def _do_table(table_element): - - elements = table_element.sub_elements - - # Our iterator index - i = float('-inf') - - def next_key(): - return t.find_following(elements, t.predicates.non_metadata, i) - - def next_assignment(): - return t.find_following(elements, t.predicates.op_assignment, next_key()) - - def next_value(): - return t.find_following(elements, t.predicates.non_metadata, next_assignment()) - - while next_key() >= 0: - - del elements[next_key()+1:next_assignment()] - del elements[next_assignment()+1:next_value()] - - elements.insert(next_assignment(), element_factory.create_whitespace_element(1)) - elements.insert(next_value(), element_factory.create_whitespace_element(1)) - - i = t.find_following(elements, t.predicates.newline, i) diff --git a/pipenv/patched/prettytoml/prettifier/tableentrysort.py b/pipenv/patched/prettytoml/prettifier/tableentrysort.py deleted file mode 100644 index 8cbd307b..00000000 --- a/pipenv/patched/prettytoml/prettifier/tableentrysort.py +++ /dev/null @@ -1,38 +0,0 @@ -import operator -from prettytoml import tokens -from prettytoml.elements.common import TokenElement -from prettytoml.elements.table import TableElement -from prettytoml.prettifier import common -from functools import * - - -def sort_table_entries(toml_file_elements): - """ - Rule: Entries within a single table should be ordered lexicographically by key - """ - return [_sorted_table(element) if isinstance(element, TableElement) else element for element in toml_file_elements] - - -def _line_key(line_elements): - """ - Given a sequence of elements comprising a single line, returns an orderable value to use in ordering lines. - """ - for e in line_elements: - if isinstance(e, TokenElement) and tokens.is_string(e.first_token): - return e.primitive_value - return 'z' * 10 # Metadata lines should be at the end - - -def _sorted_table(table): - """ - Returns another TableElement where the table entries are sorted lexicographically by key. - """ - assert isinstance(table, TableElement) - - # Discarding TokenElements with no tokens in them - table_elements = common.non_empty_elements(table.sub_elements) - lines = tuple(common.lines(table_elements)) - sorted_lines = sorted(lines, key=_line_key) - sorted_elements = reduce(operator.concat, sorted_lines) - - return TableElement(sorted_elements) diff --git a/pipenv/patched/prettytoml/prettifier/tableindent.py b/pipenv/patched/prettytoml/prettifier/tableindent.py deleted file mode 100644 index 3b60883d..00000000 --- a/pipenv/patched/prettytoml/prettifier/tableindent.py +++ /dev/null @@ -1,49 +0,0 @@ -from prettytoml import tokens -from prettytoml.elements import traversal as t, factory as element_factory -from prettytoml.tokens import py2toml - - -def table_entries_should_be_uniformly_indented(toml_file_elements): - """ - Rule: Nth-level table sections should be indented by (N-1)*2 spaces - """ - elements = toml_file_elements[:] - for (i, e) in enumerate(elements): - if t.predicates.table_header(e): - table = elements[t.find_following(elements, t.predicates.table, i)] - _do_table_header(e) - _do_table(table, len(e.names)) - return elements - - -def _do_table_header(table_header): - indent_start = 0 - indent_end = next(i for (i, token) in enumerate(table_header.tokens) if token.type != tokens.TYPE_WHITESPACE) - - del table_header.tokens[indent_start:indent_end] - table_header.tokens.insert(0, py2toml.create_whitespace(' ' * ((len(table_header.names)-1) * 2))) - - -def _do_table(table_element, table_level): - - elements = table_element.sub_elements - - # Iterator index - i = float('-inf') - - def first_indent(): - return t.find_following(elements, t.predicates.whitespace, i) - - def next_non_metadata(): - return t.find_following(elements, t.predicates.non_metadata, i) - - def next_newline(): - return t.find_following(elements, t.predicates.newline, next_non_metadata()) - - while next_non_metadata() >= 0: - if first_indent() >= 0: - del elements[first_indent():next_non_metadata()] - - elements.insert(next_non_metadata(), element_factory.create_whitespace_element((table_level-1)*2)) - - i = next_newline() diff --git a/pipenv/patched/prettytoml/prettifier/tablesep.py b/pipenv/patched/prettytoml/prettifier/tablesep.py deleted file mode 100644 index 059007f3..00000000 --- a/pipenv/patched/prettytoml/prettifier/tablesep.py +++ /dev/null @@ -1,31 +0,0 @@ - -from prettytoml.elements import traversal as t, factory as element_factory -from prettytoml.elements.metadata import WhitespaceElement, NewlineElement -from prettytoml.elements.table import TableElement - - -def table_separation(toml_file_elements): - """ - Rule: Tables should always be separated by an empty line. - """ - elements = toml_file_elements[:] - for element in elements: - if isinstance(element, TableElement): - _do_table(element.sub_elements) - return elements - - -def _do_table(table_elements): - - while table_elements and isinstance(table_elements[-1], WhitespaceElement): - del table_elements[-1] - - if not table_elements: - return - - if isinstance(table_elements[-1], NewlineElement): - last_non_metadata_i = t.find_previous(table_elements, t.predicates.non_metadata) - del table_elements[last_non_metadata_i+1:] - - table_elements.append(element_factory.create_newline_element()) - table_elements.append(element_factory.create_newline_element()) diff --git a/pipenv/patched/prettytoml/prettifier/test_commentspace.py b/pipenv/patched/prettytoml/prettifier/test_commentspace.py deleted file mode 100644 index 53d96d76..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_commentspace.py +++ /dev/null @@ -1,28 +0,0 @@ - -from .common import assert_prettifier_works -from .commentspace import comment_space - - -def test_comment_space(): - - toml_text = """ -my_key = string -id = 12 # My special ID - -[section.name] -headerk = false -# Own-line comment should stay the same -other_key = "value" -""" - - expected_toml_text = """ -my_key = string -id = 12\t# My special ID - -[section.name] -headerk = false -# Own-line comment should stay the same -other_key = "value" -""" - - assert_prettifier_works(toml_text, expected_toml_text, comment_space) diff --git a/pipenv/patched/prettytoml/prettifier/test_deindentanonymoustable.py b/pipenv/patched/prettytoml/prettifier/test_deindentanonymoustable.py deleted file mode 100644 index 10a6d2c8..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_deindentanonymoustable.py +++ /dev/null @@ -1,22 +0,0 @@ - -""" - This testing module depends on all the other modules. -""" - -from .deindentanonymoustable import deindent_anonymous_table -from .common import assert_prettifier_works - - -def test_anon_table_indent(): - toml_text = """ - key=value - another_key =44 -noname = me -""" - - expected_toml_text = """ -key=value -another_key =44 -noname = me -""" - assert_prettifier_works(toml_text, expected_toml_text, deindent_anonymous_table) diff --git a/pipenv/patched/prettytoml/prettifier/test_linelength.py b/pipenv/patched/prettytoml/prettifier/test_linelength.py deleted file mode 100644 index e4ab8fcb..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_linelength.py +++ /dev/null @@ -1,39 +0,0 @@ -from .linelength import line_length_limiter -from .common import assert_prettifier_works, elements_to_text, text_to_elements -import pytoml - - -def test_splitting_string(): - toml_text = """ -k = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. In et lectus nec erat condimentum scelerisque gravida sed ipsum. Mauris non orci tincidunt, viverra enim eget, tincidunt orci. Sed placerat nibh vitae ante maximus egestas maximus eu quam. Praesent vehicula mauris vestibulum, mattis turpis sollicitudin, aliquam felis. Pellentesque volutpat pharetra purus vel finibus. Vestibulum sed tempus dui. Maecenas auctor sit amet diam et porta. Morbi id libero at elit ultricies porta vel vitae nullam. " -""" - - expected_toml_text = """ -k = \"\"\" -Lorem ipsum dolor sit amet, consectetur adipiscing elit. In et lectus nec erat condimentum scelerisque gravida sed \\ -ipsum. Mauris non orci tincidunt, viverra enim eget, tincidunt orci. Sed placerat nibh vitae ante maximus egestas \\ -maximus eu quam. Praesent vehicula mauris vestibulum, mattis turpis sollicitudin, aliquam felis. Pellentesque volutpat \\ -pharetra purus vel finibus. Vestibulum sed tempus dui. Maecenas auctor sit amet diam et porta. Morbi id libero at elit \\ -ultricies porta vel vitae nullam. \"\"\" -""" - assert_prettifier_works(toml_text, expected_toml_text, line_length_limiter) - - -def test_splitting_array(): - toml_text = """ - -somethingweird = false - -[section] -k = [4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42] - - -[data] -id = 12 - -""" - - prettified = elements_to_text(line_length_limiter(text_to_elements(toml_text))) - - assert pytoml.loads(prettified) == pytoml.loads(toml_text) - assert all(len(line) < 120 for line in prettified.split('\n')) diff --git a/pipenv/patched/prettytoml/prettifier/test_tableassignment.py b/pipenv/patched/prettytoml/prettifier/test_tableassignment.py deleted file mode 100644 index 82fcc174..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_tableassignment.py +++ /dev/null @@ -1,29 +0,0 @@ - -from .tableassignment import table_assignment_spacing -from .common import assert_prettifier_works - - -def test_table_assignment_spacing(): - toml_text = """ - key1= "my value" - key2 =42 - keys = [4, 5,1] - - [section] - key1= "my value" - key2 =42 - keys = [4, 5,1] -""" - - expected_prettified = """ - key1 = "my value" - key2 = 42 - keys = [4, 5,1] - - [section] - key1 = "my value" - key2 = 42 - keys = [4, 5,1] -""" - - assert_prettifier_works(toml_text, expected_prettified, table_assignment_spacing) diff --git a/pipenv/patched/prettytoml/prettifier/test_tableentrysort.py b/pipenv/patched/prettytoml/prettifier/test_tableentrysort.py deleted file mode 100644 index 0cc39f78..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_tableentrysort.py +++ /dev/null @@ -1,45 +0,0 @@ - -from .tableentrysort import sort_table_entries -from .common import assert_prettifier_works - - -def test_table_sorting(): - toml_text = """description = "" -firstname = "adnan" -lastname = "fatayerji" -git_aydo = "" -groups = ["sales", "dubai", "mgmt"] -skype = "" -emails = ["adnan@incubaid.com", - "fatayera@incubaid.com", - "adnan.fatayerji@incubaid.com", - "adnan@greenitglobe.com", - "fatayera@greenitglobe.com", - "adnan.fatayerji@greenitglobe.com"] -# I really like this table -id = "fatayera" -git_github = "" -telegram = "971507192009" -mobiles = ["971507192009"] -""" - - prettified = """description = "" -emails = ["adnan@incubaid.com", - "fatayera@incubaid.com", - "adnan.fatayerji@incubaid.com", - "adnan@greenitglobe.com", - "fatayera@greenitglobe.com", - "adnan.fatayerji@greenitglobe.com"] -firstname = "adnan" -git_aydo = "" -git_github = "" -groups = ["sales", "dubai", "mgmt"] -# I really like this table -id = "fatayera" -lastname = "fatayerji" -mobiles = ["971507192009"] -skype = "" -telegram = "971507192009" -""" - - assert_prettifier_works(toml_text, prettified, sort_table_entries) diff --git a/pipenv/patched/prettytoml/prettifier/test_tableindent.py b/pipenv/patched/prettytoml/prettifier/test_tableindent.py deleted file mode 100644 index a37f73a7..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_tableindent.py +++ /dev/null @@ -1,25 +0,0 @@ - -from .tableindent import table_entries_should_be_uniformly_indented -from .common import assert_prettifier_works - - -def test_table_entries_should_be_uniformly_indented(): - toml_text = """ - [firstlevel] -hello = "my name" - my_id = 12 - - [firstlevel.secondlevel] - my_truth = False -""" - - expected_toml_text = """ -[firstlevel] -hello = "my name" -my_id = 12 - - [firstlevel.secondlevel] - my_truth = False -""" - - assert_prettifier_works(toml_text, expected_toml_text, table_entries_should_be_uniformly_indented) diff --git a/pipenv/patched/prettytoml/prettifier/test_tablesep.py b/pipenv/patched/prettytoml/prettifier/test_tablesep.py deleted file mode 100644 index a8a81d52..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_tablesep.py +++ /dev/null @@ -1,34 +0,0 @@ - -from .tablesep import table_separation -from .common import assert_prettifier_works - - -def test_table_separation(): - - toml_text = """key1 = "value1" -key2 = 22 -[section] -k = false -m= "true" - - - -[another.section] -l = "t" -creativity = "on vacation" -""" - - expected_toml_text = """key1 = "value1" -key2 = 22 - -[section] -k = false -m= "true" - -[another.section] -l = "t" -creativity = "on vacation" - -""" - - assert_prettifier_works(toml_text, expected_toml_text, table_separation) diff --git a/pipenv/patched/prettytoml/test_prettifier.py b/pipenv/patched/prettytoml/test_prettifier.py deleted file mode 100644 index f702fb01..00000000 --- a/pipenv/patched/prettytoml/test_prettifier.py +++ /dev/null @@ -1,12 +0,0 @@ - -from .prettifier import prettify -from .prettifier.common import assert_prettifier_works -import pytoml - - -def test_prettifying_against_humanly_verified_sample(): - toml_source = open('sample.toml').read() - expected = open('sample-prettified.toml').read() - - assert_prettifier_works(toml_source, expected, prettify) - assert pytoml.loads(toml_source) == pytoml.loads(expected) diff --git a/pipenv/patched/prettytoml/test_util.py b/pipenv/patched/prettytoml/test_util.py deleted file mode 100644 index b741abfa..00000000 --- a/pipenv/patched/prettytoml/test_util.py +++ /dev/null @@ -1,22 +0,0 @@ -from prettytoml.util import is_sequence_like, is_dict_like, chunkate_string - - -def test_is_sequence_like(): - assert is_sequence_like([1, 3, 4]) - assert not is_sequence_like(42) - - -def test_is_dict_like(): - assert is_dict_like({'name': False}) - assert not is_dict_like(42) - assert not is_dict_like([4, 8, 15]) - - -def test_chunkate_string(): - - text = """Lorem ipsum dolor sit amet, consectetur adipiscing elit. In et lectus nec erat condimentum scelerisque gravida sed ipsum. Mauris non orci tincidunt, viverra enim eget, tincidunt orci. Sed placerat nibh vitae ante maximus egestas maximus eu quam. Praesent vehicula mauris vestibulum, mattis turpis sollicitudin, aliquam felis. Pellentesque volutpat pharetra purus vel finibus. Vestibulum sed tempus dui. Maecenas auctor sit amet diam et porta. Morbi id libero at elit ultricies porta vel vitae nullam. """ - - chunks = chunkate_string(text, 50) - - assert ''.join(chunks) == text - assert all(len(chunk) <= 50 for chunk in chunks) diff --git a/pipenv/patched/prettytoml/tokens/__init__.py b/pipenv/patched/prettytoml/tokens/__init__.py deleted file mode 100644 index a5b2e9ae..00000000 --- a/pipenv/patched/prettytoml/tokens/__init__.py +++ /dev/null @@ -1,136 +0,0 @@ - -""" -TOML lexical tokens. -""" - -class TokenType: - """ - A TokenType is a concrete type of a source token along with a defined priority and a higher-order kind. - - The priority will be used in determining the tokenization behaviour of the lexer in the following manner: - whenever more than one token is recognizable as the next possible token and they are all of equal source - length, this priority is going to be used to break the tie by favoring the token type of the lowest priority - value. A TokenType instance is naturally ordered by its priority. - """ - - def __init__(self, name, priority, is_metadata): - self._priority = priority - self._name = name - self._is_metadata = is_metadata - - @property - def is_metadata(self): - return self._is_metadata - - @property - def priority(self): - return self._priority - - def __repr__(self): - return "{}-{}".format(self.priority, self._name) - - def __lt__(self, other): - return isinstance(other, TokenType) and self._priority < other.priority - -# Possible types of tokens -TYPE_BOOLEAN = TokenType('boolean', 0, is_metadata=False) -TYPE_INTEGER = TokenType('integer', 0, is_metadata=False) -TYPE_OP_COMMA = TokenType('comma', 0, is_metadata=True) -TYPE_OP_SQUARE_LEFT_BRACKET = TokenType('square_left_bracket', 0, is_metadata=True) -TYPE_OP_SQUARE_RIGHT_BRACKET = TokenType('square_right_bracket', 0, is_metadata=True) -TYPE_OP_CURLY_LEFT_BRACKET = TokenType('curly_left_bracket', 0, is_metadata=True) -TYPE_OP_CURLY_RIGHT_BRACKET = TokenType('curly_right_bracket', 0, is_metadata=True) -TYPE_OP_ASSIGNMENT = TokenType('assignment', 0, is_metadata=True) -TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET = TokenType('double_square_left_bracket', 0, is_metadata=True) -TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET = TokenType('double_square_right_bracket', 0, is_metadata=True) -TYPE_FLOAT = TokenType('float', 1, is_metadata=False) -TYPE_DATE = TokenType('date', 40, is_metadata=False) -TYPE_OPT_DOT = TokenType('dot', 40, is_metadata=True) -TYPE_BARE_STRING = TokenType('bare_string', 50, is_metadata=False) -TYPE_STRING = TokenType('string', 90, is_metadata=False) -TYPE_MULTILINE_STRING = TokenType('multiline_string', 90, is_metadata=False) -TYPE_LITERAL_STRING = TokenType('literal_string', 90, is_metadata=False) -TYPE_MULTILINE_LITERAL_STRING = TokenType('multiline_literal_string', 90, is_metadata=False) -TYPE_NEWLINE = TokenType('newline', 91, is_metadata=True) -TYPE_WHITESPACE = TokenType('whitespace', 93, is_metadata=True) -TYPE_COMMENT = TokenType('comment', 95, is_metadata=True) - - -def is_operator(token): - """ - Returns True if the given token is an operator token. - """ - return token.type in ( - TYPE_OP_COMMA, - TYPE_OP_SQUARE_LEFT_BRACKET, - TYPE_OP_SQUARE_RIGHT_BRACKET, - TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET, - TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET, - TYPE_OP_CURLY_LEFT_BRACKET, - TYPE_OP_CURLY_RIGHT_BRACKET, - TYPE_OP_ASSIGNMENT, - TYPE_OPT_DOT, - ) - - -def is_string(token): - return token.type in ( - TYPE_STRING, - TYPE_MULTILINE_STRING, - TYPE_LITERAL_STRING, - TYPE_BARE_STRING, - TYPE_MULTILINE_LITERAL_STRING - ) - - -class Token: - """ - A token/lexeme in a TOML source file. - - A Token instance is naturally ordered by its type. - """ - - def __init__(self, _type, source_substring, col=None, row=None): - self._source_substring = source_substring - self._type = _type - self._col = col - self._row = row - - def __eq__(self, other): - if not isinstance(other, Token): - return False - return self.source_substring == other.source_substring and self.type == other.type - - @property - def col(self): - """ - Column number (1-indexed). - """ - return self._col - - @property - def row(self): - """ - Row number (1-indexed). - """ - return self._row - - @property - def type(self): - """ - One of of the TOKEN_TYPE_* constants. - """ - return self._type - - @property - def source_substring(self): - """ - The substring of the initial source file containing this token. - """ - return self._source_substring - - def __lt__(self, other): - return isinstance(other, Token) and self.type < other.type - - def __repr__(self): - return "{}: {}".format(self.type, self.source_substring) diff --git a/pipenv/patched/prettytoml/tokens/errors.py b/pipenv/patched/prettytoml/tokens/errors.py deleted file mode 100644 index d40cb8e9..00000000 --- a/pipenv/patched/prettytoml/tokens/errors.py +++ /dev/null @@ -1,13 +0,0 @@ -from prettytoml.errors import TOMLError - - -class DeserializationError(TOMLError): - pass - - -class BadEscapeCharacter(TOMLError): - pass - - -class MalformedDateError(DeserializationError): - pass diff --git a/pipenv/patched/prettytoml/tokens/py2toml.py b/pipenv/patched/prettytoml/tokens/py2toml.py deleted file mode 100644 index 2decd021..00000000 --- a/pipenv/patched/prettytoml/tokens/py2toml.py +++ /dev/null @@ -1,154 +0,0 @@ - -""" -A converter of python values to TOML Token instances. -""" -from __future__ import unicode_literals -import codecs -import datetime -import six -from prettytoml import tokens -import re -from prettytoml.errors import TOMLError -from prettytoml.tokens import Token -from prettytoml.util import chunkate_string - - -class NotPrimitiveError(TOMLError): - pass - - -_operator_tokens_by_type = { - tokens.TYPE_OP_SQUARE_LEFT_BRACKET: tokens.Token(tokens.TYPE_OP_SQUARE_LEFT_BRACKET, u'['), - tokens.TYPE_OP_SQUARE_RIGHT_BRACKET: tokens.Token(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET, u']'), - tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET: tokens.Token(tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET, u'[['), - tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET: tokens.Token(tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET, u']]'), - tokens.TYPE_OP_COMMA: tokens.Token(tokens.TYPE_OP_COMMA, u','), - tokens.TYPE_NEWLINE: tokens.Token(tokens.TYPE_NEWLINE, u'\n'), - tokens.TYPE_OPT_DOT: tokens.Token(tokens.TYPE_OPT_DOT, u'.'), -} - - -def operator_token(token_type): - return _operator_tokens_by_type[token_type] - - -def create_primitive_token(value, multiline_strings_allowed=True): - """ - Creates and returns a single token for the given primitive atomic value. - - Raises NotPrimitiveError when the given value is not a primitive atomic value - """ - if value is None: - return create_primitive_token('') - elif isinstance(value, bool): - return tokens.Token(tokens.TYPE_BOOLEAN, u'true' if value else u'false') - elif isinstance(value, int): - return tokens.Token(tokens.TYPE_INTEGER, u'{}'.format(value)) - elif isinstance(value, float): - return tokens.Token(tokens.TYPE_FLOAT, u'{}'.format(value)) - elif isinstance(value, (datetime.datetime, datetime.date, datetime.time)): - s = value.isoformat() - if s.endswith('+00:00'): - s = s[:-6] + 'Z' - return tokens.Token(tokens.TYPE_DATE, s) - elif isinstance(value, six.string_types): - return create_string_token(value, multiline_strings_allowed=multiline_strings_allowed) - - raise NotPrimitiveError("{} of type {}".format(value, type(value))) - - -_bare_string_regex = re.compile('^[a-zA-Z_-]*$') - - -def create_string_token(text, bare_string_allowed=False, multiline_strings_allowed=True): - """ - Creates and returns a single string token. - - Raises ValueError on non-string input. - """ - - if not isinstance(text, six.string_types): - raise ValueError('Given value must be a string') - - if text == '': - return tokens.Token(tokens.TYPE_STRING, '""'.format(_escape_single_line_quoted_string(text))) - elif bare_string_allowed and _bare_string_regex.match(text): - return tokens.Token(tokens.TYPE_BARE_STRING, text) - elif multiline_strings_allowed and (len(tuple(c for c in text if c == '\n')) >= 2 or len(text) > 80): - # If containing two or more newlines or is longer than 80 characters we'll use the multiline string format - return _create_multiline_string_token(text) - else: - return tokens.Token(tokens.TYPE_STRING, '"{}"'.format(_escape_single_line_quoted_string(text))) - - -def _escape_single_line_quoted_string(text): - text = text.decode('utf-8') if isinstance(text, six.binary_type) else text - start = 0 - i = 0 - res = [] - _escapes = {'\n': '\\n', '\r': '\\r', '\\': '\\\\', '\t': '\\t', - '\b': '\\b', '\f': '\\f', '"': '\\"'} - - def flush(): - if start < i: - res.append(text[start:i]) - return i + 1 - - while i < len(text): - c = text[i] - if c in _escapes: - start = flush() - res.append(_escapes[c]) - elif ord(c) < 0x20: - start = flush() - res.append('\\u%04x' % ord(c)) - i += 1 - - flush() - return ''.join(res) - - -def _create_multiline_string_token(text): - escaped = text.replace(u'"""', u'\"\"\"') - if len(escaped) > 50: - return tokens.Token(tokens.TYPE_MULTILINE_STRING, u'"""\n{}\\\n"""'.format(_break_long_text(escaped))) - else: - return tokens.Token(tokens.TYPE_MULTILINE_STRING, u'"""{}"""'.format(escaped)) - - -def _break_long_text(text, maximum_length=75): - """ - Breaks into lines of 75 character maximum length that are terminated by a backslash. - """ - - def next_line(remaining_text): - - # Returns a line and the remaining text - - if '\n' in remaining_text and remaining_text.index('\n') < maximum_length: - i = remaining_text.index('\n') - return remaining_text[:i+1], remaining_text[i+2:] - elif len(remaining_text) > maximum_length and ' ' in remaining_text: - i = remaining_text[:maximum_length].rfind(' ') - return remaining_text[:i+1] + '\\\n', remaining_text[i+2:] - else: - return remaining_text, '' - - remaining_text = text - lines = [] - while remaining_text: - line, remaining_text = next_line(remaining_text) - lines += [line] - - return ''.join(lines) - - -def create_whitespace(source_substring): - return Token(tokens.TYPE_WHITESPACE, source_substring) - - -def create_multiline_string(text, maximum_line_length=120): - def escape(t): - return t.replace(u'"""', six.u(r'\"\"\"')) - source_substring = u'"""\n{}"""'.format(u'\\\n'.join(chunkate_string(escape(text), maximum_line_length))) - return Token(tokens.TYPE_MULTILINE_STRING, source_substring) diff --git a/pipenv/patched/prettytoml/tokens/test_py2toml.py b/pipenv/patched/prettytoml/tokens/test_py2toml.py deleted file mode 100644 index 0d029c5d..00000000 --- a/pipenv/patched/prettytoml/tokens/test_py2toml.py +++ /dev/null @@ -1,69 +0,0 @@ -import datetime - -import strict_rfc3339 - -from prettytoml import tokens -from prettytoml.tokens import py2toml - - -def test_string(): - assert py2toml.create_string_token('fawzy', bare_string_allowed=True) == tokens.Token(tokens.TYPE_BARE_STRING, 'fawzy') - assert py2toml.create_primitive_token('I am a "cr\'azy" sentence.') == \ - tokens.Token(tokens.TYPE_STRING, '"I am a \\"cr\'azy\\" sentence."') - - -def test_multiline_string(): - text = 'The\nSuper\nT"""OML"""\n\nIs coming' - - primitive_token = py2toml.create_primitive_token(text) - - assert primitive_token.source_substring == '"""The\nSuper\nT\"\"\"OML\"\"\"\n\nIs coming"""' - - -def test_long_string(): - text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Suspendisse faucibus nibh id urna euismod, " \ - "vitae blandit nisi blandit. Nam eu odio ex. Praesent iaculis sapien justo. Proin vehicula orci rhoncus " \ - "risus mattis cursus. Sed quis commodo diam. Morbi dictum fermentum ex. Ut augue lorem, facilisis eu " \ - "posuere ut, ullamcorper et quam. Donec porta neque eget erat lacinia, in convallis elit scelerisque. " \ - "Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Praesent " \ - "felis metus, venenatis eu aliquam vel, fringilla in turpis. Praesent interdum pulvinar enim, et mattis " \ - "urna dapibus et. Sed ut egestas mauris. Etiam eleifend dui." - - primitive_token = py2toml.create_primitive_token(text) - - assert primitive_token.source_substring[3:-3] == r""" -Lorem ipsum dolor sit amet, consectetur adipiscing elit. Suspendisse \ -aucibus nibh id urna euismod, vitae blandit nisi blandit. Nam eu odio ex. \ -raesent iaculis sapien justo. Proin vehicula orci rhoncus risus mattis \ -ursus. Sed quis commodo diam. Morbi dictum fermentum ex. Ut augue lorem, \ -acilisis eu posuere ut, ullamcorper et quam. Donec porta neque eget erat \ -acinia, in convallis elit scelerisque. Class aptent taciti sociosqu ad \ -itora torquent per conubia nostra, per inceptos himenaeos. Praesent felis \ -etus, venenatis eu aliquam vel, fringilla in turpis. Praesent interdum \ -ulvinar enim, et mattis urna dapibus et. Sed ut egestas mauris. Etiam \ -leifend dui.\ -""" - - -def test_int(): - assert py2toml.create_primitive_token(42) == tokens.Token(tokens.TYPE_INTEGER, '42') - - -def test_float(): - assert py2toml.create_primitive_token(4.2) == tokens.Token(tokens.TYPE_FLOAT, '4.2') - - -def test_bool(): - assert py2toml.create_primitive_token(False) == tokens.Token(tokens.TYPE_BOOLEAN, 'false') - assert py2toml.create_primitive_token(True) == tokens.Token(tokens.TYPE_BOOLEAN, 'true') - - -def test_date(): - ts = strict_rfc3339.rfc3339_to_timestamp('1979-05-27T00:32:00-07:00') - dt = datetime.datetime.fromtimestamp(ts) - assert py2toml.create_primitive_token(dt) == tokens.Token(tokens.TYPE_DATE, '1979-05-27T07:32:00Z') - - -def test_none(): - t = py2toml.create_primitive_token(None) - assert t.type == tokens.TYPE_STRING and t.source_substring == '""' diff --git a/pipenv/patched/prettytoml/tokens/test_toml2py.py b/pipenv/patched/prettytoml/tokens/test_toml2py.py deleted file mode 100644 index ce166365..00000000 --- a/pipenv/patched/prettytoml/tokens/test_toml2py.py +++ /dev/null @@ -1,86 +0,0 @@ -from datetime import datetime - -import pytz - -from prettytoml import tokens -from prettytoml.tokens import toml2py -from prettytoml.tokens.errors import BadEscapeCharacter, DeserializationError - - -def test_integer(): - t1 = tokens.Token(tokens.TYPE_INTEGER, '42') - t2 = tokens.Token(tokens.TYPE_INTEGER, '1_001_2') - - assert toml2py.deserialize(t1) == 42 - assert toml2py.deserialize(t2) == 10012 - - -def test_float(): - tokens_and_values = ( - ('4.2', 4.2), - ('12e2', 12e2), - ('1_000e2', 1e5), - ('314.1e-2', 3.141) - ) - for token_string, value in tokens_and_values: - token = tokens.Token(tokens.TYPE_FLOAT, token_string) - assert toml2py.deserialize(token) == value - - -def test_string(): - - t0 = tokens.Token(tokens.TYPE_BARE_STRING, 'fawzy') - assert toml2py.deserialize(t0) == 'fawzy' - - t1 = tokens.Token(tokens.TYPE_STRING, '"I\'m a string. \\"You can quote me\\". Name\\tJos\\u00E9\\nLocation\\tSF."') - assert toml2py.deserialize(t1) == u'I\'m a string. "You can quote me". Name\tJos\xe9\nLocation\tSF.' - - t2 = tokens.Token(tokens.TYPE_MULTILINE_STRING, '"""\nRoses are red\nViolets are blue"""') - assert toml2py.deserialize(t2) == 'Roses are red\nViolets are blue' - - t3_str = '"""\nThe quick brown \\\n\n\n fox jumps over \\\n the lazy dog."""' - t3 = tokens.Token(tokens.TYPE_MULTILINE_STRING, t3_str) - assert toml2py.deserialize(t3) == 'The quick brown fox jumps over the lazy dog.' - - t4_str = '"""\\\n The quick brown \\\n fox jumps over \\\n the lazy dog.\\\n """' - t4 = tokens.Token(tokens.TYPE_MULTILINE_STRING, t4_str) - assert toml2py.deserialize(t4) == 'The quick brown fox jumps over the lazy dog.' - - t5 = tokens.Token(tokens.TYPE_LITERAL_STRING, r"'C:\Users\nodejs\templates'") - assert toml2py.deserialize(t5) == r'C:\Users\nodejs\templates' - - t6_str = "'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''" - t6 = tokens.Token(tokens.TYPE_MULTILINE_LITERAL_STRING, t6_str) - assert toml2py.deserialize(t6) == 'The first newline is\ntrimmed in raw strings.\n All' \ - ' other whitespace\n is preserved.\n' - - -def test_date(): - t0 = tokens.Token(tokens.TYPE_DATE, '1979-05-27T07:32:00Z') - assert toml2py.deserialize(t0) == datetime(1979, 5, 27, 7, 32, tzinfo=pytz.utc) - - t1 = tokens.Token(tokens.TYPE_DATE, '1979-05-27T00:32:00-07:00') - assert toml2py.deserialize(t1) == datetime(1979, 5, 27, 7, 32, tzinfo=pytz.utc) - - t3 = tokens.Token(tokens.TYPE_DATE, '1987-07-05T17:45:00') - try: - toml2py.deserialize(t3) - assert False, 'Should detect malformed date' - except DeserializationError: - pass - - -def test_unescaping_a_string(): - - bad_escapes = ( - r"This string has a bad \a escape character.", - r'\x33', - ) - - for source in bad_escapes: - # Should complain about bad escape jobs - try: - toml2py._unescape_str(source) - assert False, "Should have thrown an exception for: " + source - except BadEscapeCharacter: - pass diff --git a/pipenv/patched/prettytoml/tokens/toml2py.py b/pipenv/patched/prettytoml/tokens/toml2py.py deleted file mode 100644 index 56804437..00000000 --- a/pipenv/patched/prettytoml/tokens/toml2py.py +++ /dev/null @@ -1,130 +0,0 @@ -from __future__ import unicode_literals -import re -import string -import iso8601 -from prettytoml import tokens -from prettytoml.tokens import TYPE_BOOLEAN, TYPE_INTEGER, TYPE_FLOAT, TYPE_DATE, \ - TYPE_MULTILINE_STRING, TYPE_BARE_STRING, TYPE_MULTILINE_LITERAL_STRING, TYPE_LITERAL_STRING, \ - TYPE_STRING -import codecs -import six -from prettytoml.tokens.errors import MalformedDateError -from .errors import BadEscapeCharacter -import functools -import operator - - -def deserialize(token): - """ - Deserializes the value of a single tokens.Token instance based on its type. - - Raises DeserializationError when appropriate. - """ - - if token.type == TYPE_BOOLEAN: - return _to_boolean(token) - elif token.type == TYPE_INTEGER: - return _to_int(token) - elif token.type == TYPE_FLOAT: - return _to_float(token) - elif token.type == TYPE_DATE: - return _to_date(token) - elif token.type in (TYPE_STRING, TYPE_MULTILINE_STRING, TYPE_BARE_STRING, - TYPE_LITERAL_STRING, TYPE_MULTILINE_LITERAL_STRING): - return _to_string(token) - else: - raise Exception('This should never happen!') - - -def _unescape_str(text): - """ - Unescapes a string according the TOML spec. Raises BadEscapeCharacter when appropriate. - """ - text = text.decode('utf-8') if isinstance(text, six.binary_type) else text - tokens = [] - i = 0 - basicstr_re = re.compile(r'[^"\\\000-\037]*') - unicode_re = re.compile(r'[uU]((?<=u)[a-fA-F0-9]{4}|(?<=U)[a-fA-F0-9]{8})') - escapes = { - 'b': '\b', - 't': '\t', - 'n': '\n', - 'f': '\f', - 'r': '\r', - '\\': '\\', - '"': '"', - '/': '/', - "'": "'" - } - while True: - m = basicstr_re.match(text, i) - i = m.end() - tokens.append(m.group()) - if i == len(text) or text[i] != '\\': - break - else: - i += 1 - if unicode_re.match(text, i): - m = unicode_re.match(text, i) - i = m.end() - tokens.append(six.unichr(int(m.group(1), 16))) - else: - if text[i] not in escapes: - raise BadEscapeCharacter - tokens.append(escapes[text[i]]) - i += 1 - return ''.join(tokens) - - -def _to_string(token): - if token.type == tokens.TYPE_BARE_STRING: - return token.source_substring - - elif token.type == tokens.TYPE_STRING: - escaped = token.source_substring[1:-1] - return _unescape_str(escaped) - - elif token.type == tokens.TYPE_MULTILINE_STRING: - escaped = token.source_substring[3:-3] - - # Drop the first newline if existed - if escaped and escaped[0] == '\n': - escaped = escaped[1:] - - # Remove all occurrences of a slash-newline-zero-or-more-whitespace patterns - escaped = re.sub(r'\\\n\s*', repl='', string=escaped, flags=re.DOTALL) - return _unescape_str(escaped) - - elif token.type == tokens.TYPE_LITERAL_STRING: - return token.source_substring[1:-1] - - elif token.type == tokens.TYPE_MULTILINE_LITERAL_STRING: - text = token.source_substring[3:-3] - if text[0] == '\n': - text = text[1:] - return text - - raise RuntimeError('Control should never reach here.') - - -def _to_int(token): - return int(token.source_substring.replace('_', '')) - - -def _to_float(token): - assert token.type == tokens.TYPE_FLOAT - string = token.source_substring.replace('_', '') - return float(string) - - -def _to_boolean(token): - return token.source_substring == 'true' - - -_correct_date_format = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|(\+|-)\d{2}:\d{2})') - - -def _to_date(token): - if not _correct_date_format.match(token.source_substring): - raise MalformedDateError - return iso8601.parse_date(token.source_substring) diff --git a/pipenv/patched/prettytoml/util.py b/pipenv/patched/prettytoml/util.py deleted file mode 100644 index 155f00e2..00000000 --- a/pipenv/patched/prettytoml/util.py +++ /dev/null @@ -1,141 +0,0 @@ -import math -import itertools - - -def is_sequence_like(x): - """ - Returns True if x exposes a sequence-like interface. - """ - required_attrs = ( - '__len__', - '__getitem__' - ) - return all(hasattr(x, attr) for attr in required_attrs) - - -def is_dict_like(x): - """ - Returns True if x exposes a dict-like interface. - """ - required_attrs = ( - '__len__', - '__getitem__', - 'keys', - 'values', - ) - return all(hasattr(x, attr) for attr in required_attrs) - - -def join_with(iterable, separator): - """ - Joins elements from iterable with separator and returns the produced sequence as a list. - - separator must be addable to a list. - """ - inputs = list(iterable) - b = [] - for i, element in enumerate(inputs): - if isinstance(element, (list, tuple, set)): - b += tuple(element) - else: - b += [element] - if i < len(inputs)-1: - b += separator - return b - - -def chunkate_string(text, length): - """ - Iterates over the given seq in chunks of at maximally the given length. Will never break a whole word. - """ - iterator_index = 0 - - def next_newline(): - try: - return next(i for (i, c) in enumerate(text) if i > iterator_index and c == '\n') - except StopIteration: - return len(text) - - def next_breaker(): - try: - return next(i for (i, c) in reversed(tuple(enumerate(text))) - if i >= iterator_index and - (i < iterator_index+length) and - c in (' ', '\t')) - except StopIteration: - return len(text) - - while iterator_index < len(text): - next_chunk = text[iterator_index:min(next_newline(), next_breaker()+1)] - iterator_index += len(next_chunk) - yield next_chunk - - -def flatten_nested(nested_dicts): - """ - Flattens dicts and sequences into one dict with tuples of keys representing the nested keys. - - Example - >>> dd = { \ - 'dict1': {'name': 'Jon', 'id': 42}, \ - 'dict2': {'name': 'Sam', 'id': 41}, \ - 'seq1': [{'one': 1, 'two': 2}] \ - } - - >>> flatten_nested(dd) == { \ - ('dict1', 'name'): 'Jon', ('dict1', 'id'): 42, \ - ('dict2', 'name'): 'Sam', ('dict2', 'id'): 41, \ - ('seq1', 0, 'one'): 1, ('seq1', 0, 'two'): 2, \ - } - True - """ - assert isinstance(nested_dicts, (dict, list, tuple)), 'Only works with a collection parameter' - - def items(c): - if isinstance(c, dict): - return c.items() - elif isinstance(c, (list, tuple)): - return enumerate(c) - else: - raise RuntimeError('c must be a collection') - - def flatten(dd): - output = {} - for k, v in items(dd): - if isinstance(v, (dict, list, tuple)): - for child_key, child_value in flatten(v).items(): - output[(k,) + child_key] = child_value - else: - output[(k,)] = v - return output - - return flatten(nested_dicts) - - -class PeekableIterator: - - # Returned by peek() when the iterator is exhausted. Truthiness is False. - Nothing = tuple() - - def __init__(self, iter): - self._iter = iter - - def __next__(self): - return next(self._iter) - - def next(self): - return self.__next__() - - def __iter__(self): - return self - - def peek(self): - """ - Returns PeekableIterator.Nothing when the iterator is exhausted. - """ - try: - v = next(self._iter) - self._iter = itertools.chain((v,), self._iter) - return v - except StopIteration: - return PeekableIterator.Nothing diff --git a/pipenv/project.py b/pipenv/project.py index 4b90f493..13c6619f 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -75,7 +75,7 @@ class _LockFileEncoder(json.JSONEncoder): This adds a few characteristics to the encoder: * The JSON is always prettified with indents and spaces. - * PrettyTOML's container elements are seamlessly encodable. + * TOMLKit's container elements are seamlessly encodable. * The output is always UTF-8-encoded text, never binary, even on Python 2. """ @@ -85,11 +85,7 @@ class _LockFileEncoder(json.JSONEncoder): ) def default(self, obj): - from prettytoml.elements.common import ContainerElement, TokenElement - - if isinstance(obj, (ContainerElement, TokenElement)): - return obj.primitive_value - elif isinstance(obj, vistir.compat.Path): + if isinstance(obj, vistir.compat.Path): obj = obj.as_posix() return super(_LockFileEncoder, self).default(obj) diff --git a/tasks/vendoring/patches/patched/contoml.patch b/tasks/vendoring/patches/patched/contoml.patch deleted file mode 100644 index b9b2e9d7..00000000 --- a/tasks/vendoring/patches/patched/contoml.patch +++ /dev/null @@ -1,28 +0,0 @@ -diff --git a/pipenv/patched/contoml/file/file.py b/pipenv/patched/contoml/file/file.py -index 5033a7b..99ce148 100644 ---- a/pipenv/patched/contoml/file/file.py -+++ b/pipenv/patched/contoml/file/file.py -@@ -30,6 +30,14 @@ class TOMLFile: - except KeyError: - return FreshTable(parent=self, name=item, is_array=False) - -+ def get(self, item, default=None): -+ """This was not here for who knows why.""" -+ -+ if item not in self: -+ return default -+ else: -+ return self.__getitem__(item) -+ - def __contains__(self, item): - return item in self.keys() - -@@ -223,7 +231,7 @@ class TOMLFile: - if has_anonymous_entry(): - return items - else: -- return items + [('', self[''])] -+ return list(items) + [('', self[''])] - - @property - def primitive(self): diff --git a/tasks/vendoring/patches/patched/prettytoml-newlinefix.patch b/tasks/vendoring/patches/patched/prettytoml-newlinefix.patch deleted file mode 100644 index 2b1066a1..00000000 --- a/tasks/vendoring/patches/patched/prettytoml-newlinefix.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/pipenv/patched/prettytoml/elements/traversal/__init__.py b/pipenv/patched/prettytoml/elements/traversal/__init__.py -index 5b98045..c93506e 100644 ---- a/pipenv/patched/prettytoml/elements/traversal/__init__.py -+++ b/pipenv/patched/prettytoml/elements/traversal/__init__.py -@@ -78,7 +78,7 @@ class TraversalMixin: - - if following_comment == float('-inf'): - return following_newline -- if following_newline == float('inf'): -+ if following_newline == float('-inf'): - return following_comment - - if following_newline < following_comment: diff --git a/tasks/vendoring/patches/patched/prettytoml-python37.patch b/tasks/vendoring/patches/patched/prettytoml-python37.patch deleted file mode 100644 index 5039a1c7..00000000 --- a/tasks/vendoring/patches/patched/prettytoml-python37.patch +++ /dev/null @@ -1,32 +0,0 @@ -From c44f2126fb5c75a5f5afd9d320c9f6cfc4ce3384 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= -Date: Tue, 26 Jun 2018 21:02:45 +0200 -Subject: [PATCH] Catch StopIteration in AbstractTable._enumerate_items - -This makes PEP 479 enabled Pythons (such as 3.7) work again. - -Otherwise you get: - - RuntimeError: generator raised StopIteration - -Fixes https://github.com/pypa/pipenv/issues/2426 ---- - pipenv/patched/prettytoml/elements/abstracttable.py | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/pipenv/patched/prettytoml/elements/abstracttable.py b/pipenv/patched/prettytoml/elements/abstracttable.py -index 59fd574..627da0e 100644 ---- a/pipenv/patched/prettytoml/elements/abstracttable.py -+++ b/pipenv/patched/prettytoml/elements/abstracttable.py -@@ -19,7 +19,10 @@ def _enumerate_items(self): - """ - non_metadata = self._enumerate_non_metadata_sub_elements() - while True: -- yield next(non_metadata), next(non_metadata) -+ try: -+ yield next(non_metadata), next(non_metadata) -+ except StopIteration: -+ return - - def items(self): - for (key_i, key), (value_i, value) in self._enumerate_items(): diff --git a/tasks/vendoring/patches/patched/prettytoml-table-iter.patch b/tasks/vendoring/patches/patched/prettytoml-table-iter.patch deleted file mode 100644 index 9ec52633..00000000 --- a/tasks/vendoring/patches/patched/prettytoml-table-iter.patch +++ /dev/null @@ -1,29 +0,0 @@ -diff --git a/pipenv/patched/prettytoml/elements/abstracttable.py b/pipenv/patched/prettytoml/elements/abstracttable.py -index 59fd5748..48663aed 100644 ---- a/pipenv/patched/prettytoml/elements/abstracttable.py -+++ b/pipenv/patched/prettytoml/elements/abstracttable.py -@@ -1,8 +1,13 @@ -+try: -+ from collections.abc import Mapping -+except ImportError: -+ from collections import Mapping -+ - from prettytoml.elements.common import ContainerElement - from prettytoml.elements import traversal - - --class AbstractTable(ContainerElement, traversal.TraversalMixin): -+class AbstractTable(ContainerElement, traversal.TraversalMixin, Mapping): - """ - Common code for handling tables as key-value pairs with metadata elements sprinkled all over. - -@@ -37,6 +42,9 @@ class AbstractTable(ContainerElement, traversal.TraversalMixin): - def __len__(self): - return len(tuple(self._enumerate_items())) - -+ def __iter__(self): -+ return (key for key, _ in self.items()) -+ - def __contains__(self, item): - return item in self.keys() - diff --git a/tasks/vendoring/patches/patched/prettytoml-unicode.patch b/tasks/vendoring/patches/patched/prettytoml-unicode.patch deleted file mode 100644 index 54f4c621..00000000 --- a/tasks/vendoring/patches/patched/prettytoml-unicode.patch +++ /dev/null @@ -1,132 +0,0 @@ -diff --git a/pipenv/patched/prettytoml/tokens/py2toml.py b/pipenv/patched/prettytoml/tokens/py2toml.py -index 8299195..2decd02 100644 ---- a/pipenv/patched/prettytoml/tokens/py2toml.py -+++ b/pipenv/patched/prettytoml/tokens/py2toml.py -@@ -2,6 +2,7 @@ - """ - A converter of python values to TOML Token instances. - """ -+from __future__ import unicode_literals - import codecs - import datetime - import six -@@ -81,10 +82,30 @@ def create_string_token(text, bare_string_allowed=False, multiline_strings_allow - - - def _escape_single_line_quoted_string(text): -- if six.PY2: -- return text.encode('unicode-escape').encode('string-escape').replace('"', '\\"').replace("\\'", "'") -- else: -- return codecs.encode(text, 'unicode-escape').decode().replace('"', '\\"') -+ text = text.decode('utf-8') if isinstance(text, six.binary_type) else text -+ start = 0 -+ i = 0 -+ res = [] -+ _escapes = {'\n': '\\n', '\r': '\\r', '\\': '\\\\', '\t': '\\t', -+ '\b': '\\b', '\f': '\\f', '"': '\\"'} -+ -+ def flush(): -+ if start < i: -+ res.append(text[start:i]) -+ return i + 1 -+ -+ while i < len(text): -+ c = text[i] -+ if c in _escapes: -+ start = flush() -+ res.append(_escapes[c]) -+ elif ord(c) < 0x20: -+ start = flush() -+ res.append('\\u%04x' % ord(c)) -+ i += 1 -+ -+ flush() -+ return ''.join(res) - - - def _create_multiline_string_token(text): -diff --git a/pipenv/patched/prettytoml/tokens/toml2py.py b/pipenv/patched/prettytoml/tokens/toml2py.py -index 2bf9c1c..5680443 100644 ---- a/pipenv/patched/prettytoml/tokens/toml2py.py -+++ b/pipenv/patched/prettytoml/tokens/toml2py.py -@@ -1,3 +1,4 @@ -+from __future__ import unicode_literals - import re - import string - import iso8601 -@@ -39,42 +40,40 @@ def _unescape_str(text): - """ - Unescapes a string according the TOML spec. Raises BadEscapeCharacter when appropriate. - """ -- -- # Detect bad escape jobs -- bad_escape_regexp = re.compile(r'([^\\]|^)\\[^btnfr"\\uU]') -- if bad_escape_regexp.findall(text): -- raise BadEscapeCharacter -- -- # Do the unescaping -- if six.PY2: -- return _unicode_escaped_string(text).decode('string-escape').decode('unicode-escape') -- else: -- return codecs.decode(_unicode_escaped_string(text), 'unicode-escape') -- -- --def _unicode_escaped_string(text): -- """ -- Escapes all unicode characters in the given string -- """ -- -- if six.PY2: -- text = unicode(text) -- -- def is_unicode(c): -- return c.lower() not in string.ascii_letters + string.whitespace + string.punctuation + string.digits -- -- def escape_unicode_char(x): -- if six.PY2: -- return x.encode('unicode-escape') -+ text = text.decode('utf-8') if isinstance(text, six.binary_type) else text -+ tokens = [] -+ i = 0 -+ basicstr_re = re.compile(r'[^"\\\000-\037]*') -+ unicode_re = re.compile(r'[uU]((?<=u)[a-fA-F0-9]{4}|(?<=U)[a-fA-F0-9]{8})') -+ escapes = { -+ 'b': '\b', -+ 't': '\t', -+ 'n': '\n', -+ 'f': '\f', -+ 'r': '\r', -+ '\\': '\\', -+ '"': '"', -+ '/': '/', -+ "'": "'" -+ } -+ while True: -+ m = basicstr_re.match(text, i) -+ i = m.end() -+ tokens.append(m.group()) -+ if i == len(text) or text[i] != '\\': -+ break - else: -- return codecs.encode(x, 'unicode-escape') -- -- if any(is_unicode(c) for c in text): -- homogeneous_chars = tuple(escape_unicode_char(c) if is_unicode(c) else c.encode() for c in text) -- homogeneous_bytes = functools.reduce(operator.add, homogeneous_chars) -- return homogeneous_bytes.decode() -- else: -- return text -+ i += 1 -+ if unicode_re.match(text, i): -+ m = unicode_re.match(text, i) -+ i = m.end() -+ tokens.append(six.unichr(int(m.group(1), 16))) -+ else: -+ if text[i] not in escapes: -+ raise BadEscapeCharacter -+ tokens.append(escapes[text[i]]) -+ i += 1 -+ return ''.join(tokens) - - - def _to_string(token): diff --git a/tasks/vendoring/patches/patched/prettytoml.patch b/tasks/vendoring/patches/patched/prettytoml.patch deleted file mode 100644 index 85dfb791..00000000 --- a/tasks/vendoring/patches/patched/prettytoml.patch +++ /dev/null @@ -1,78 +0,0 @@ -diff --git a/pipenv/patched/prettytoml/_version.py b/pipenv/patched/prettytoml/_version.py -index 4f146e6..e0f1547 100644 ---- a/pipenv/patched/prettytoml/_version.py -+++ b/pipenv/patched/prettytoml/_version.py -@@ -1 +1 @@ --VERSION = '0.03' -+VERSION = 'master' -diff --git a/pipenv/patched/prettytoml/elements/table.py b/pipenv/patched/prettytoml/elements/table.py -index f78a6d1..cdc3ed4 100644 ---- a/pipenv/patched/prettytoml/elements/table.py -+++ b/pipenv/patched/prettytoml/elements/table.py -@@ -94,9 +94,9 @@ class TableElement(abstracttable.AbstractTable): - value_element, - factory.create_newline_element(), - ] -- -+ - insertion_index = self._find_insertion_index() -- -+ - self._sub_elements = \ - self.sub_elements[:insertion_index] + inserted_elements + self.sub_elements[insertion_index:] - -@@ -105,11 +105,16 @@ class TableElement(abstracttable.AbstractTable): - preceding_newline = self._find_preceding_newline(begin) - if preceding_newline >= 0: - begin = preceding_newline -- end = self._find_following_newline(begin) -+ end = self._find_following_line_terminator(begin) - if end < 0: - end = len(tuple(self._sub_elements)) - self._sub_elements = self.sub_elements[:begin] + self.sub_elements[end:] - -+ def pop(self, key): -+ v = self[key] -+ del self[key] -+ return v -+ - def value(self): - return self - -diff --git a/pipenv/patched/prettytoml/tokens/py2toml.py b/pipenv/patched/prettytoml/tokens/py2toml.py -index 3db97b4..8299195 100644 ---- a/pipenv/patched/prettytoml/tokens/py2toml.py -+++ b/pipenv/patched/prettytoml/tokens/py2toml.py -@@ -5,11 +5,8 @@ A converter of python values to TOML Token instances. - import codecs - import datetime - import six --import strict_rfc3339 --import timestamp - from prettytoml import tokens - import re --from prettytoml.elements.metadata import NewlineElement - from prettytoml.errors import TOMLError - from prettytoml.tokens import Token - from prettytoml.util import chunkate_string -@@ -49,15 +46,17 @@ def create_primitive_token(value, multiline_strings_allowed=True): - elif isinstance(value, float): - return tokens.Token(tokens.TYPE_FLOAT, u'{}'.format(value)) - elif isinstance(value, (datetime.datetime, datetime.date, datetime.time)): -- ts = timestamp(value) // 1000 -- return tokens.Token(tokens.TYPE_DATE, strict_rfc3339.timestamp_to_rfc3339_utcoffset(ts)) -+ s = value.isoformat() -+ if s.endswith('+00:00'): -+ s = s[:-6] + 'Z' -+ return tokens.Token(tokens.TYPE_DATE, s) - elif isinstance(value, six.string_types): - return create_string_token(value, multiline_strings_allowed=multiline_strings_allowed) - - raise NotPrimitiveError("{} of type {}".format(value, type(value))) - - --_bare_string_regex = re.compile('^[a-zA-Z0-9_-]*$') -+_bare_string_regex = re.compile('^[a-zA-Z_-]*$') - - - def create_string_token(text, bare_string_allowed=False, multiline_strings_allowed=True): diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 1e00bbb7..5e1bafb9 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -160,4 +160,4 @@ version = "*" with open(p.pipfile_path) as f: contents = f.read() assert "[packages.requests]" not in contents - assert 'requests = { version = "*" }' in contents + assert 'requests = {version = "*"}' in contents diff --git a/tests/unit/test_vendor.py b/tests/unit/test_vendor.py index 704b37fb..6514b162 100644 --- a/tests/unit/test_vendor.py +++ b/tests/unit/test_vendor.py @@ -1,43 +1,10 @@ # -*- coding: utf-8 -*- # We need to import the patched packages directly from sys.path, so the # identity checks can pass. -import pipenv # noqa - -import datetime import os -import pytest -import pytz - -import contoml +import pipenv # noqa from pipfile.api import PipfileParser -from prettytoml import lexer, tokens -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import ( - WhitespaceElement, PunctuationElement, CommentElement -) -from prettytoml.elements.table import TableElement -from prettytoml.tokens.py2toml import create_primitive_token - - -def test_table(): - initial_toml = """id=42 # My id\nage=14""" - tokens = tuple(lexer.tokenize(initial_toml)) - table = TableElement( - [ - AtomicElement(tokens[0:1]), - PunctuationElement(tokens[1:2]), - AtomicElement(tokens[2:3]), - WhitespaceElement(tokens[3:4]), - CommentElement(tokens[4:6]), - AtomicElement(tokens[6:7]), - PunctuationElement(tokens[7:8]), - AtomicElement(tokens[8:9]), - ] - ) - assert set(table.items()) == {('id', 42), ('age', 14)} - del table['id'] - assert set(table.items()) == {('age', 14)} class TestPipfileParser: @@ -71,38 +38,3 @@ class TestPipfileParser: assert parsed_dict["list"][1] == {} assert parsed_dict["bool"] is True assert parsed_dict["none"] is None - - -@pytest.mark.parametrize('dt, content', [ - ( # Date. - datetime.date(1992, 8, 19), - '1992-08-19', - ), - ( # Naive time. - datetime.time(15, 10), - '15:10:00', - ), - ( # Aware time in UTC. - datetime.time(15, 10, tzinfo=pytz.UTC), - '15:10:00Z', - ), - ( # Aware local time. - datetime.time(15, 10, tzinfo=pytz.FixedOffset(8 * 60)), - '15:10:00+08:00', - ), - ( # Naive datetime. - datetime.datetime(1992, 8, 19, 15, 10), - '1992-08-19T15:10:00', - ), - ( # Aware datetime in UTC. - datetime.datetime(1992, 8, 19, 15, 10, tzinfo=pytz.UTC), - '1992-08-19T15:10:00Z', - ), - ( # Aware local datetime. - datetime.datetime(1992, 8, 19, 15, 10, tzinfo=pytz.FixedOffset(8 * 60)), - '1992-08-19T15:10:00+08:00', - ), -]) -def test_token_date(dt, content): - token = create_primitive_token(dt) - assert token == tokens.Token(tokens.TYPE_DATE, content) From b27d6a771081f7f88dd3d0005e0d60bbbf0ee53d Mon Sep 17 00:00:00 2001 From: frostming Date: Thu, 8 Nov 2018 11:04:42 +0800 Subject: [PATCH 114/218] clear references in patched.txt --- pipenv/patched/patched.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index 4f3ee409..e7dadd8e 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -1,5 +1,4 @@ safety -git+https://github.com/jumpscale7/python-consistent-toml.git#egg=contoml crayons==0.1.2 pipfile==0.0.2 pip-tools==3.1.0 From 78be826034fcc9d67398e023e594af02005b553b Mon Sep 17 00:00:00 2001 From: jxltom Date: Thu, 8 Nov 2018 11:10:48 +0800 Subject: [PATCH 115/218] Fix typo --- pipenv/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 758ef3fa..b57d9fa3 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -454,8 +454,8 @@ def ensure_python(three=None, python=None): click.echo(fix_utf8("Something went wrong…"), err=True) click.echo(crayons.blue(e.err), err=True) else: - environments.PIPENV_SPINNER_OK_TEXT.format("Success!") - # Print the results, in a beautiful blue… + sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + # Print the results, in a beautiful blue… click.echo(crayons.blue(c.out), err=True) # Find the newly installed Python, hopefully. version = str(version) From 9c02f6ef5f301a62eb1574fb6ecbb819239faacb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 22:16:14 -0500 Subject: [PATCH 116/218] Update vcs resolution Signed-off-by: Dan Ryan --- pipenv/core.py | 58 ++++++++++++++++++++++++------------------------- pipenv/utils.py | 8 ++++--- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 5316733c..ec2c1a82 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -969,7 +969,6 @@ def do_lock( pypi_mirror=None, ): """Executes the freeze functionality.""" - from .utils import get_vcs_deps cached_lockfile = {} if not pre: @@ -1931,38 +1930,39 @@ def do_install( extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) - except (ValueError, RuntimeError): + except (ValueError, RuntimeError) as e: sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) - # Warn if --editable wasn't passed. - if pkg_requirement.is_vcs and not pkg_requirement.editable: - sp.write_err( - "{0}: You installed a VCS dependency in non-editable mode. " - "This will work fine, but sub-dependencies will not be resolved by {1}." - "\n To enable this sub-dependency functionality, specify that this dependency is editable." - "".format( - crayons.red("Warning", bold=True), - crayons.red("$ pipenv lock"), - ) - ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. - if c.return_code != 0: - sp.write_err(vistir.compat.fs_str( - "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(pkg_line) - ), - )) - sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) - if "setup.py egg_info" in c.err: - sp.write_err(vistir.compat.fs_str( - "This is likely caused by a bug in {0}. " - "Report this to its maintainers.".format( - crayons.green(pkg_requirement.name) + else: + # Warn if --editable wasn't passed. + if pkg_requirement.is_vcs and not pkg_requirement.editable: + sp.write_err( + "{0}: You installed a VCS dependency in non-editable mode. " + "This will work fine, but sub-dependencies will not be resolved by {1}." + "\n To enable this sub-dependency functionality, specify that this dependency is editable." + "".format( + crayons.red("Warning", bold=True), + crayons.red("$ pipenv lock"), ) + ) + click.echo(crayons.blue(format_pip_output(c.out))) + # Ensure that package was successfully installed. + if c.return_code != 0: + sp.write_err(vistir.compat.fs_str( + "{0} An error occurred while installing {1}!".format( + crayons.red("Error: ", bold=True), crayons.green(pkg_line) + ), )) - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) - sys.exit(1) + sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) + if "setup.py egg_info" in c.err: + sp.write_err(vistir.compat.fs_str( + "This is likely caused by a bug in {0}. " + "Report this to its maintainers.".format( + crayons.green(pkg_requirement.name) + ) + )) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sys.exit(1) sp.write(vistir.compat.fs_str( u"{0} {1} {2} {3}{4}".format( crayons.normal(u"Adding", bold=True), diff --git a/pipenv/utils.py b/pipenv/utils.py index 444333de..45e37920 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -518,7 +518,7 @@ def venv_resolve_deps( dev=False, ): from .vendor.vistir.misc import fs_str - from .vendor.vistir.compat import Path, JSONDecodeError + from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError from .vendor.vistir.path import create_tracked_tempdir from . import resolver import json @@ -533,7 +533,7 @@ def venv_resolve_deps( vcs_section = "vcs_dev_packages" if dev else "vcs_packages" if getattr(project, vcs_section, []): with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: - vcs_reqs, vcs_lockfile = get_vcs_deps( + vcs_deps, vcs_lockfile = get_vcs_deps( project, which=which, clear=clear, @@ -541,7 +541,8 @@ def venv_resolve_deps( allow_global=allow_global, dev=dev, ) - vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] + vcs_deps = [req.as_line() for req in vcs_deps if req.editable] + deps.extend([req.as_line() for req in vcs_deps if not req.editable]) sp.write(environments.PIPENV_SPINNER_OK_TEXT.format( "Successfully pinned VCS Packages!" )) @@ -569,6 +570,7 @@ def venv_resolve_deps( if vcs_deps: with temp_environ(): os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) + sp.text = to_native_string("Locking VCS Dependencies...") vcs_c = resolve(cmd, sp) vcs_results, vcs_err = vcs_c.out, vcs_c.err else: From 5fbfe6a177ceab63e2a03f8bed1d363400fb7607 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 8 Nov 2018 00:27:11 -0500 Subject: [PATCH 117/218] Refactor locking Signed-off-by: Dan Ryan --- pipenv/core.py | 39 ++++++++-------------------------- pipenv/utils.py | 56 ++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 57 insertions(+), 38 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index ec2c1a82..9dbadd76 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1027,7 +1027,8 @@ def do_lock( deps = convert_deps_to_pip( settings["packages"], project, r=False, include_index=True ) - results, vcs_results = venv_resolve_deps( + lockfile_base = lockfile[settings["lockfile_key"]].copy() + locked_lockfile = venv_resolve_deps( deps, which=which, project=project, @@ -1036,33 +1037,10 @@ def do_lock( pre=pre, allow_global=system, pypi_mirror=pypi_mirror, + pipfile=settings["packages"], + lockfile=lockfile_base ) - vcs_results, vcs_lockfile = vcs_results - # Add dependencies to lockfile. - for dep in results: - is_top_level = dep["name"] in settings["packages"] - pipfile_entry = settings["packages"][dep["name"]] if is_top_level else None - dep_lockfile = clean_resolved_dep( - dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry - ) - lockfile[settings["lockfile_key"]].update(dep_lockfile) - for dep in vcs_results: - normalized = pep423_name(dep["name"]) - if not hasattr(dep, "keys") or not hasattr(dep["name"], "keys"): - continue - is_top_level = dep["name"] in vcs_lockfile or normalized in vcs_lockfile - if is_top_level: - try: - pipfile_entry = vcs_lockfile[dep["name"]] - except KeyError: - pipfile_entry = vcs_lockfile[normalized] - else: - pipfile_entry = None - dep_lockfile = clean_resolved_dep( - dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry - ) - vcs_lockfile.update(dep_lockfile) - lockfile[settings["lockfile_key"]].update(vcs_lockfile) + lockfile[settings["lockfile_key"]] = locked_lockfile # Support for --keep-outdated… if keep_outdated: @@ -1079,9 +1057,10 @@ def do_lock( section_name ][canonical_name].copy() # Overwrite any develop packages with default packages. - for default_package in lockfile["default"]: - if default_package in lockfile["develop"]: - lockfile["develop"][default_package] = lockfile["default"][default_package] + develop_keys = set(list(lockfile["develop"].keys())) + default_keys = set(list(lockfile["default"].keys())) + for pkg in default_keys & develop_keys: + lockfile["develop"][pkg] = lockfile["default"][pkg] if write: project.write_lockfile(lockfile) click.echo( diff --git a/pipenv/utils.py b/pipenv/utils.py index 45e37920..bd3e2754 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -507,6 +507,36 @@ def resolve(cmd, sp): return c +def get_locked_dep(dep, pipfile_section): + entry = None + lockfile_entry = None + if isinstance(dep, Mapping) and dep.get("name", ""): + name_options = [dep.get("name"), pep423_name(dep.get("name"))] + name = next(iter(k for k in name_options if k in pipfile_section), None) + entry = pipfile_section.get(name, None) + lockfile_entry = clean_resolved_dep(dep, is_top_level=True, pipfile_entry=entry) + else: + lockfile_entry = clean_resolved_dep(dep, is_top_level=False, pipfile_entry=entry) + return lockfile_entry + + +def prepare_lockfiles(results, pipfile, lockfile, vcs_lockfile): + from .vendor.requirementslib.utils import is_vcs + for dep in results: + # Merge in any relevant information from the pipfile entry, including + # markers, normalized names, URL info, etc that we may have dropped during lock + if not is_vcs(dep): + lockfile_entry = get_locked_dep(dep, pipfile) + lockfile.update(lockfile_entry) + # For vcs dependencies, treat the initial pass at locking (i.e. checkout) + # as the pipfile entry because it gets us an actual ref to use + else: + lockfile_entry = get_locked_dep(dep, vcs_lockfile) + vcs_lockfile.update(lockfile_entry) + lockfile.update(vcs_lockfile) + return lockfile + + def venv_resolve_deps( deps, which, @@ -516,6 +546,8 @@ def venv_resolve_deps( allow_global=False, pypi_mirror=None, dev=False, + pipfile=None, + lockfile=None ): from .vendor.vistir.misc import fs_str from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError @@ -526,14 +558,21 @@ def venv_resolve_deps( vcs_deps = [] vcs_lockfile = {} results = [] - if not deps: - return results, (vcs_deps, vcs_lockfile) + pipfile_section = "dev_packages" if dev else "packages" + lockfile_section = "develop" if dev else "default" + vcs_section = "vcs_{0}".format(pipfile_section) + vcs_deps = getattr(project, vcs_section, []) + if not deps and not vcs_deps: + return {} + if not pipfile: + pipfile = getattr(project, pipfile_section, None) + if not lockfile: + lockfile = project._lockfile[lockfile_section] req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") - vcs_section = "vcs_dev_packages" if dev else "vcs_packages" - if getattr(project, vcs_section, []): + if vcs_deps: with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: - vcs_deps, vcs_lockfile = get_vcs_deps( + vcs_reqs, vcs_lockfile = get_vcs_deps( project, which=which, clear=clear, @@ -541,8 +580,7 @@ def venv_resolve_deps( allow_global=allow_global, dev=dev, ) - vcs_deps = [req.as_line() for req in vcs_deps if req.editable] - deps.extend([req.as_line() for req in vcs_deps if not req.editable]) + vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] sp.write(environments.PIPENV_SPINNER_OK_TEXT.format( "Successfully pinned VCS Packages!" )) @@ -592,7 +630,9 @@ def venv_resolve_deps( click_echo(out.strip(), err=True) click_echo(err.strip(), err=True) raise RuntimeError("There was a problem with locking.") - return results, (vcs_results, vcs_lockfile) + results += vcs_results + lockfile = prepare_lockfiles(results, pipfile, lockfile, vcs_lockfile) + return lockfile def resolve_deps( From 1555200463cd91babc1197e81bec15357d7f11f4 Mon Sep 17 00:00:00 2001 From: frostming Date: Thu, 8 Nov 2018 14:59:59 +0800 Subject: [PATCH 118/218] Add back some tests --- tests/unit/test_vendor.py | 49 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_vendor.py b/tests/unit/test_vendor.py index 6514b162..35458c53 100644 --- a/tests/unit/test_vendor.py +++ b/tests/unit/test_vendor.py @@ -1,9 +1,15 @@ # -*- coding: utf-8 -*- # We need to import the patched packages directly from sys.path, so the # identity checks can pass. +import pipenv # noqa + +import datetime import os -import pipenv # noqa +import pytest +import pytz +import tomlkit + from pipfile.api import PipfileParser @@ -38,3 +44,44 @@ class TestPipfileParser: assert parsed_dict["list"][1] == {} assert parsed_dict["bool"] is True assert parsed_dict["none"] is None + + +@pytest.mark.parametrize('dt, content', [ + ( # Date. + datetime.date(1992, 8, 19), + '1992-08-19', + ), + ( # Naive time. + datetime.time(15, 10), + '15:10:00', + ), + ( # Aware time in UTC. + datetime.time(15, 10, tzinfo=pytz.UTC), + '15:10:00+00:00', + ), + ( # Aware local time. + datetime.time(15, 10, tzinfo=pytz.FixedOffset(8 * 60)), + '15:10:00+08:00', + ), + ( # Naive datetime. + datetime.datetime(1992, 8, 19, 15, 10), + '1992-08-19T15:10:00', + ), + ( # Aware datetime in UTC. + datetime.datetime(1992, 8, 19, 15, 10, tzinfo=pytz.UTC), + '1992-08-19T15:10:00Z', + ), + ( # Aware local datetime. + datetime.datetime(1992, 8, 19, 15, 10, tzinfo=pytz.FixedOffset(8 * 60)), + '1992-08-19T15:10:00+08:00', + ), +]) +def test_token_date(dt, content): + item = tomlkit.item(dt) + assert item.as_string() == content + + +def test_dump_nonascii_string(): + content = 'name = "Stažené"\n' + toml_content = tomlkit.dumps(tomlkit.loads(content)) + assert toml_content == content From 494b7f3e9f8f3d5edb6406997fe3c8d95ce5239e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 8 Nov 2018 16:17:41 +0900 Subject: [PATCH 119/218] Make the PIP_SHIMS_BASE_MODULE pop actually work --- pipenv/core.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 758ef3fa..f9b8eaee 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1949,9 +1949,13 @@ def do_install( extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) - except (ValueError, RuntimeError): - sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + except (ValueError, RuntimeError) as e: + sp.write_err(vistir.compat.fs_str( + "{0}: {1}".format(crayons.red("WARNING"), e), + )) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Installation Failed", + )) # Warn if --editable wasn't passed. if pkg_requirement.is_vcs and not pkg_requirement.editable: sp.write_err( @@ -2179,9 +2183,15 @@ def do_uninstall( def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror=None): # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) + ensure_project( + three=three, python=python, validate=False, pypi_mirror=pypi_mirror, + ) + # Set an environment variable, so we know we're in the environment. os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") + + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + # Support shell compatibility mode. if PIPENV_SHELL_FANCY: fancy = True @@ -2191,9 +2201,12 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror shell = choose_shell() click.echo(fix_utf8("Launching subshell in virtual environment…"), err=True) - fork_args = (project.virtualenv_location, project.project_directory, shell_args) - with vistir.contextmanagers.temp_environ(): - os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + fork_args = ( + project.virtualenv_location, + project.project_directory, + shell_args, + ) + if fancy: shell.fork(*fork_args) return @@ -2322,14 +2335,17 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): from .cmdparse import ScriptEmptyError # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) + ensure_project( + three=three, python=python, validate=False, pypi_mirror=pypi_mirror, + ) + # Set an environment variable, so we know we're in the environment. os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") - load_dot_env() - # Activate virtualenv under the current interpreter's environment - with vistir.contextmanagers.temp_environ(): - os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + load_dot_env() + + # Activate virtualenv under the current interpreter's environment inline_activate_virtual_environment() try: script = project.build_script(command, args) From 53b073c7ffcce7d18dd3b459932d49f783546434 Mon Sep 17 00:00:00 2001 From: frostming Date: Thu, 8 Nov 2018 15:20:47 +0800 Subject: [PATCH 120/218] python 2.7 unicode --- pipenv/project.py | 2 +- tests/unit/test_vendor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 13c6619f..74fe6527 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -856,7 +856,7 @@ class Project(object): formatted_data = tomlkit.dumps(data).rstrip() else: encoder = toml.encoder.TomlPreserveInlineDictEncoder() - formatted_data = toml.dumps(data, encoder=encoder) + formatted_data = toml.dumps(data, encoder=encoder).rstrip() except Exception: document = tomlkit.document() for section in ("packages", "dev-packages"): diff --git a/tests/unit/test_vendor.py b/tests/unit/test_vendor.py index 35458c53..aea93112 100644 --- a/tests/unit/test_vendor.py +++ b/tests/unit/test_vendor.py @@ -82,6 +82,6 @@ def test_token_date(dt, content): def test_dump_nonascii_string(): - content = 'name = "Stažené"\n' + content = u'name = "Stažené"\n' toml_content = tomlkit.dumps(tomlkit.loads(content)) assert toml_content == content From 3bc76bc9497978bf8314013e6bb5e3f70321915f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 8 Nov 2018 03:41:29 -0500 Subject: [PATCH 121/218] A bit more refactoring and cleanup Signed-off-by: Dan Ryan --- pipenv/core.py | 57 +++++++++++++++++++------------------------------ pipenv/utils.py | 34 ++++++++++++++--------------- 2 files changed, 39 insertions(+), 52 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 9dbadd76..b20358ef 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -959,6 +959,14 @@ def get_downloads_info(names_map, section): return info +def overwrite_dev(prod, dev): + dev_keys = set(list(dev.keys())) + prod_keys = set(list(prod.keys())) + for pkg in dev_keys & prod_keys: + dev[pkg] = prod[pkg] + return dev + + def do_lock( ctx=None, system=False, @@ -989,58 +997,40 @@ def do_lock( del lockfile[section][k] # Ensure that develop inherits from default. dev_packages = project.dev_packages.copy() - for dev_package in project.dev_packages: - if dev_package in project.packages: - dev_packages[dev_package] = project.packages[dev_package] + dev_packages = overwrite_dev(project.packages, dev_packages) # Resolve dev-package dependencies, with pip-tools. - sections = { - "dev": { - "packages": project.dev_packages, - "vcs": project.vcs_dev_packages, - "pipfile_key": "dev_packages", - "lockfile_key": "develop", - "log_string": "dev-packages", - "dev": True, - }, - "default": { - "packages": project.packages, - "vcs": project.vcs_packages, - "pipfile_key": "packages", - "lockfile_key": "default", - "log_string": "packages", - "dev": False, - }, - } - for section_name in ["dev", "default"]: - settings = sections[section_name] + for is_dev in [True, False]: + pipfile_section = "dev_packages" if is_dev else "packages" + lockfile_section = "develop" if is_dev else "default" + packages = getattr(project, pipfile_section) + if write: # Alert the user of progress. click.echo( u"{0} {1} {2}".format( crayons.normal(u"Locking"), - crayons.red(u"[{0}]".format(settings["log_string"])), + crayons.red(u"[{0}]".format(pipfile_section.replace("_", "-"))), crayons.normal(fix_utf8("dependencies…")), ), err=True, ) deps = convert_deps_to_pip( - settings["packages"], project, r=False, include_index=True + packages, project, r=False, include_index=True ) - lockfile_base = lockfile[settings["lockfile_key"]].copy() - locked_lockfile = venv_resolve_deps( + # Mutates the lockfile + venv_resolve_deps( deps, which=which, project=project, - dev=settings["dev"], + dev=is_dev, clear=clear, pre=pre, allow_global=system, pypi_mirror=pypi_mirror, - pipfile=settings["packages"], - lockfile=lockfile_base + pipfile=packages, + lockfile=lockfile ) - lockfile[settings["lockfile_key"]] = locked_lockfile # Support for --keep-outdated… if keep_outdated: @@ -1057,10 +1047,7 @@ def do_lock( section_name ][canonical_name].copy() # Overwrite any develop packages with default packages. - develop_keys = set(list(lockfile["develop"].keys())) - default_keys = set(list(lockfile["default"].keys())) - for pkg in default_keys & develop_keys: - lockfile["develop"][pkg] = lockfile["default"][pkg] + lockfile["develop"].update(overwrite_dev(lockfile.get("default", {}), lockfile["develop"])) if write: project.write_lockfile(lockfile) click.echo( diff --git a/pipenv/utils.py b/pipenv/utils.py index bd3e2754..93026ca7 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -509,18 +509,22 @@ def resolve(cmd, sp): def get_locked_dep(dep, pipfile_section): entry = None - lockfile_entry = None + cleaner_kwargs = { + "is_top_level": False, + "pipfile_entry": None + } if isinstance(dep, Mapping) and dep.get("name", ""): - name_options = [dep.get("name"), pep423_name(dep.get("name"))] + name_options = [dep["name"], pep423_name(dep["name"])] name = next(iter(k for k in name_options if k in pipfile_section), None) - entry = pipfile_section.get(name, None) - lockfile_entry = clean_resolved_dep(dep, is_top_level=True, pipfile_entry=entry) - else: - lockfile_entry = clean_resolved_dep(dep, is_top_level=False, pipfile_entry=entry) + entry = pipfile_section[name] if name else None + + if entry: + cleaner_kwargs.update({"is_top_level": True, "pipfile_entry": entry}) + lockfile_entry = clean_resolved_dep(dep, **cleaner_kwargs) return lockfile_entry -def prepare_lockfiles(results, pipfile, lockfile, vcs_lockfile): +def prepare_lockfile(results, pipfile, lockfile): from .vendor.requirementslib.utils import is_vcs for dep in results: # Merge in any relevant information from the pipfile entry, including @@ -528,12 +532,6 @@ def prepare_lockfiles(results, pipfile, lockfile, vcs_lockfile): if not is_vcs(dep): lockfile_entry = get_locked_dep(dep, pipfile) lockfile.update(lockfile_entry) - # For vcs dependencies, treat the initial pass at locking (i.e. checkout) - # as the pipfile entry because it gets us an actual ref to use - else: - lockfile_entry = get_locked_dep(dep, vcs_lockfile) - vcs_lockfile.update(lockfile_entry) - lockfile.update(vcs_lockfile) return lockfile @@ -568,7 +566,7 @@ def venv_resolve_deps( if not pipfile: pipfile = getattr(project, pipfile_section, None) if not lockfile: - lockfile = project._lockfile[lockfile_section] + lockfile = project._lockfile req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") if vcs_deps: with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: @@ -621,7 +619,10 @@ def venv_resolve_deps( try: results = json.loads(results.split("RESULTS:")[1].strip()) if vcs_results: + # For vcs dependencies, treat the initial pass at locking (i.e. checkout) + # as the pipfile entry because it gets us an actual ref to use vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) + vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile) else: vcs_results = [] @@ -630,9 +631,8 @@ def venv_resolve_deps( click_echo(out.strip(), err=True) click_echo(err.strip(), err=True) raise RuntimeError("There was a problem with locking.") - results += vcs_results - lockfile = prepare_lockfiles(results, pipfile, lockfile, vcs_lockfile) - return lockfile + lockfile[lockfile_section] = prepare_lockfile(results, pipfile, lockfile[lockfile_section]) + lockfile[lockfile_section].update(vcs_lockfile) def resolve_deps( From a0b8c6d34a5c9358ff65df6254fd79e4ff72ef18 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 8 Nov 2018 03:52:12 -0500 Subject: [PATCH 122/218] Fix syntax error Signed-off-by: Dan Ryan --- pipenv/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipenv/core.py b/pipenv/core.py index 6a492eed..cbc43d3a 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1913,6 +1913,7 @@ def do_install( crayons.red("Warning", bold=True), crayons.red("$ pipenv lock"), ) + ) click.echo(crayons.blue(format_pip_output(c.out))) # Ensure that package was successfully installed. if c.return_code != 0: From b3aa66b1542200994dcdd5770228568a7f6dbac7 Mon Sep 17 00:00:00 2001 From: frostming Date: Thu, 8 Nov 2018 17:26:39 +0800 Subject: [PATCH 123/218] make tomlkit dump toml's inline table --- pipenv/project.py | 13 +++++----- pipenv/vendor/tomlkit/items.py | 6 ++++- .../vendor/tomlkit-dump-inline-table.patch | 24 +++++++++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch diff --git a/pipenv/project.py b/pipenv/project.py index 74fe6527..6d59b4cb 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -572,10 +572,15 @@ class Project(object): """Clear pipfile cache (e.g., so we can mutate parsed pipfile)""" _pipfile_cache.clear() + @staticmethod + def _is_tomlkit_parsed_result(parsed): + """Check by duck typing of tomlkit.api.Container""" + return hasattr(parsed, "_body") + @staticmethod def convert_outline_table(parsed): """Converts all outline to inline tables""" - if hasattr(parsed, "_body"): # Duck-type that implies tomlkit.api.Container. + if Project._istomlkit_parsed_result(parsed): empty_inline_table = tomlkit.inline_table else: empty_inline_table = toml.TomlDecoder().get_empty_inline_table @@ -852,11 +857,7 @@ class Project(object): if path is None: path = self.pipfile_location try: - if hasattr(data, "_body"): - formatted_data = tomlkit.dumps(data).rstrip() - else: - encoder = toml.encoder.TomlPreserveInlineDictEncoder() - formatted_data = toml.dumps(data, encoder=encoder).rstrip() + formatted_data = tomlkit.dumps(data).rstrip() except Exception: document = tomlkit.document() for section in ("packages", "dev-packages"): diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 781e2e98..80e029d9 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -21,6 +21,7 @@ if PY2: from pipenv.vendor.backports.functools_lru_cache import lru_cache else: from functools import lru_cache +from toml.decoder import InlineTableDict def item(value, _parent=None): @@ -36,7 +37,10 @@ def item(value, _parent=None): elif isinstance(value, float): return Float(value, Trivia(), str(value)) elif isinstance(value, dict): - val = Table(Container(), Trivia(), False) + if isinstance(value, InlineTableDict): + val = InlineTable(Container(), Trivia()) + else: + val = Table(Container(), Trivia(), False) for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): val[k] = item(v, _parent=val) diff --git a/tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch b/tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch new file mode 100644 index 00000000..755bd31a --- /dev/null +++ b/tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch @@ -0,0 +1,24 @@ +diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py +index 781e2e98..80e029d9 100644 +--- a/pipenv/vendor/tomlkit/items.py ++++ b/pipenv/vendor/tomlkit/items.py +@@ -21,6 +21,7 @@ if PY2: + from pipenv.vendor.backports.functools_lru_cache import lru_cache + else: + from functools import lru_cache ++from toml.decoder import InlineTableDict + + + def item(value, _parent=None): +@@ -36,7 +37,10 @@ def item(value, _parent=None): + elif isinstance(value, float): + return Float(value, Trivia(), str(value)) + elif isinstance(value, dict): +- val = Table(Container(), Trivia(), False) ++ if isinstance(value, InlineTableDict): ++ val = InlineTable(Container(), Trivia()) ++ else: ++ val = Table(Container(), Trivia(), False) + for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): + val[k] = item(v, _parent=val) + From 36f054d3a81790918a9300808c4302f247a31685 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 8 Nov 2018 04:47:26 -0500 Subject: [PATCH 124/218] Grab updates from latest vendored changes Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/models/pyenv.py | 7 +- pipenv/vendor/pythonfinder/models/python.py | 2 +- pipenv/vendor/requirementslib/__init__.py | 2 +- .../requirementslib/models/baserequirement.py | 37 --------- .../vendor/requirementslib/models/markers.py | 6 +- .../vendor/requirementslib/models/pipfile.py | 39 +++++++-- .../requirementslib/models/requirements.py | 81 +++++++++++++++---- pipenv/vendor/requirementslib/models/utils.py | 5 +- pipenv/vendor/vistir/spin.py | 2 + 10 files changed, 113 insertions(+), 70 deletions(-) delete mode 100644 pipenv/vendor/requirementslib/models/baserequirement.py diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 0b22546a..85666b5c 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.7' +__version__ = '1.1.8' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 4a8dfc65..ac7f8588 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -45,13 +45,16 @@ class PyenvFinder(BaseFinder, BasePath): ) def get_version_order(self): - version_order_file = self.root.joinpath("version").read_text(encoding="utf-8") + version_order_file, version_order_lines = self.root.joinpath("version"), [] + if version_order_file.exists(): + version_order_lines = version_order_file.read_text(encoding="utf-8").splitlines() + version_paths = [ p for p in self.root.glob("versions/*") if not (p.parent.name == "envs" or p.name == "envs") ] versions = {v.name: v for v in version_paths} - version_order = [versions[v] for v in version_order_file.splitlines() if v in versions] + version_order = [versions[v] for v in version_order_lines if v in versions] for version in version_order: version_paths.remove(version) version_order += version_paths diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index ec99afe7..24d520b6 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -173,7 +173,7 @@ class PythonVersion(object): def get_architecture(self): if self.architecture: return self.architecture - arch, _ = platform.architecture(path.path.as_posix()) + arch, _ = platform.architecture(self.comes_from.path.as_posix()) self.architecture = arch return self.architecture diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index ba0ce9ae..edbab5bc 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.2.5' +__version__ = '1.2.6' import logging diff --git a/pipenv/vendor/requirementslib/models/baserequirement.py b/pipenv/vendor/requirementslib/models/baserequirement.py deleted file mode 100644 index b97dee40..00000000 --- a/pipenv/vendor/requirementslib/models/baserequirement.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import -import abc -import attr -import six - - -@six.add_metaclass(abc.ABCMeta) -class BaseRequirement: - @classmethod - def from_line(cls, line): - """Returns a requirement from a requirements.txt or pip-compatible line""" - raise NotImplementedError - - @abc.abstractmethod - def line_part(self): - """Returns the current requirement as a pip-compatible line""" - - @classmethod - def from_pipfile(cls, name, pipfile): - """Returns a requirement from a pipfile entry""" - raise NotImplementedError - - @abc.abstractmethod - def pipfile_part(self): - """Returns the current requirement as a pipfile entry""" - - @classmethod - def attr_fields(cls): - return [field.name for field in attr.fields(cls)] - - @property - def extras_as_pip(self): - if self.extras: - return "[{0}]".format(",".join(self.extras)) - - return "" diff --git a/pipenv/vendor/requirementslib/models/markers.py b/pipenv/vendor/requirementslib/models/markers.py index 83b44b63..70fe3bc0 100644 --- a/pipenv/vendor/requirementslib/models/markers.py +++ b/pipenv/vendor/requirementslib/models/markers.py @@ -4,12 +4,11 @@ import attr from packaging.markers import InvalidMarker, Marker from ..exceptions import RequirementError -from .baserequirement import BaseRequirement from .utils import filter_none, validate_markers @attr.s -class PipenvMarkers(BaseRequirement): +class PipenvMarkers(object): """System-level requirements - see PEP508 for more detail""" os_name = attr.ib( @@ -78,7 +77,8 @@ class PipenvMarkers(BaseRequirement): @classmethod def from_pipfile(cls, name, pipfile): - found_keys = [k for k in pipfile.keys() if k in cls.attr_fields()] + attr_fields = [field.name for field in attr.fields(cls)] + found_keys = [k for k in pipfile.keys() if k in attr_fields] marker_strings = ["{0} {1}".format(k, pipfile[k]) for k in found_keys] if pipfile.get("markers"): marker_strings.append(pipfile.get("markers")) diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index fe7743c2..58d54055 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -38,12 +38,16 @@ class PipfileLoader(plette.pipfiles.Pipfile): content = content.decode(encoding) _data = tomlkit.loads(content) if "source" not in _data: - # HACK: There is no good way to prepend a section to an existing - # TOML document, but there's no good way to copy non-structural - # content from one TOML document to another either. Modify the - # TOML content directly, and load the new in-memory document. - sep = "" if content.startswith("\n") else "\n" - content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content + if "sources" in _data: + _data["source"] = _data["sources"] + content = tomlkit.dumps(_data) + else: + # HACK: There is no good way to prepend a section to an existing + # TOML document, but there's no good way to copy non-structural + # content from one TOML document to another either. Modify the + # TOML content directly, and load the new in-memory document. + sep = "" if content.startswith("\n") else "\n" + content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content data = tomlkit.loads(content) return cls(data) @@ -53,6 +57,8 @@ class Pipfile(object): path = attr.ib(validator=is_path, type=Path) projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) _pipfile = attr.ib(type=plette.pipfiles.Pipfile) + _pyproject = attr.ib(default=attr.Factory(tomlkit.document), type=tomlkit.toml_document.TOMLDocument) + build_system = attr.ib(default=attr.Factory(dict), type=dict) requirements = attr.ib(default=attr.Factory(list), type=list) dev_requirements = attr.ib(default=attr.Factory(list), type=list) @@ -212,3 +218,24 @@ class Pipfile(object): if as_requirements: return self.requirements return self._pipfile.get('packages', {}) + + def _read_pyproject(self): + pyproject = self.path.parent.joinpath("pyproject.toml") + if pyproject.exists(): + self._pyproject = tomlkit.load(pyproject) + build_system = self._pyproject.get("build-system", None) + if not os.path.exists(self.path_to("setup.py")): + if not build_system or not build_system.get("requires"): + build_system = { + "requires": ["setuptools>=38.2.5", "wheel"], + "build-backend": "setuptools.build_meta", + } + self._build_system = build_system + + @property + def build_requires(self): + return self.build_system.get("requires", []) + + @property + def build_backend(self): + return self.build_system.get("build-backend", None) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 8d087d23..ce2b0927 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -36,7 +36,6 @@ from ..utils import ( add_ssh_scheme_to_git_uri, strip_ssh_from_git_uri, ) -from .baserequirement import BaseRequirement from .utils import ( HASH_STRING, build_vcs_link, @@ -61,7 +60,7 @@ from .utils import ( @attr.s(slots=True) -class NamedRequirement(BaseRequirement): +class NamedRequirement(object): name = attr.ib() version = attr.ib(validator=attr.validators.optional(validate_specifiers)) req = attr.ib() @@ -98,7 +97,8 @@ class NamedRequirement(BaseRequirement): def from_pipfile(cls, name, pipfile): creation_args = {} if hasattr(pipfile, "keys"): - creation_args = {k: v for k, v in pipfile.items() if k in cls.attr_fields()} + attr_fields = [field.name for field in attr.fields(cls)] + creation_args = {k: v for k, v in pipfile.items() if k in attr_fields} creation_args["name"] = name version = get_version(pipfile) extras = creation_args.get("extras", None) @@ -131,7 +131,7 @@ LinkInfo = collections.namedtuple( @attr.s(slots=True) -class FileRequirement(BaseRequirement): +class FileRequirement(object): """File requirements for tar.gz installable files or wheels or setup.py containing directories.""" @@ -152,6 +152,8 @@ class FileRequirement(BaseRequirement): name = attr.ib() #: A :class:`~pkg_resources.Requirement` isntance req = attr.ib() + #: Whether this is a direct url requirement + is_direct = attr.ib(default=False) _uri_scheme = attr.ib(default=None) @classmethod @@ -256,11 +258,17 @@ class FileRequirement(BaseRequirement): return LinkInfo(vcs_type, prefer, relpath, path, uri, link) + def __attrs_post_init__(self): + if self.req and getattr(self.req, "url"): + self.uri = self.req.url + @uri.default def get_uri(self): if self.path and not self.uri: self._uri_scheme = "path" self.uri = pip_shims.shims.path_to_url(os.path.abspath(self.path)) + elif self.req and getattr(self.req, "url"): + self.uri = self.req.url @name.default def get_name(self): @@ -268,6 +276,8 @@ class FileRequirement(BaseRequirement): if loc: self._uri_scheme = "path" if self.path else "uri" name = None + if self.req and getattr(self.req, "name"): + return self.req.name if self.link and self.link.egg_fragment: return self.link.egg_fragment elif self.link and self.link.is_wheel: @@ -326,9 +336,18 @@ class FileRequirement(BaseRequirement): @req.default def get_requirement(self): - req = init_requirement(normalize_name(self.name)) - req.editable = False - req.line = self.link.url_without_fragment + if self.link.is_artifact and not self.editable: + if self._uri_scheme == "uri": + if self.name: + req_str = "{0} @{1}".format(self.name, self.link.url_without_fragment) + else: + req_str = "{0}".format(self.link.url_without_fragment) + req = init_requirement(req_str) + req.line = req_str + else: + req = init_requirement(normalize_name(self.name)) + req.editable = False + req.line = self.link.url_without_fragment if self.path and self.link and self.link.scheme.startswith("file"): req.local_file = True req.path = self.path @@ -337,7 +356,8 @@ class FileRequirement(BaseRequirement): else: req.local_file = False req.path = None - req.url = self.link.url_without_fragment + if not getattr(req, "url", None): + req.url = self.link.url_without_fragment if self.editable: req.editable = True req.link = self.link @@ -351,9 +371,13 @@ class FileRequirement(BaseRequirement): for scheme in ("http", "https", "ftp", "ftps", "uri") ) and (self.link.is_artifact or self.link.is_wheel) - and not self.req.editable + and not self.editable ) + @property + def is_direct_url(self): + return self.is_remote_artifact + @property def formatted_path(self): if self.path: @@ -371,10 +395,18 @@ class FileRequirement(BaseRequirement): editable = line.startswith("-e ") line = line.split(" ", 1)[1] if editable else line setup_path = None + name = None + req = None if not any([is_installable_file(line), is_valid_url(line), is_file_url(line)]): - raise RequirementError( - "Supplied requirement is not installable: {0!r}".format(line) - ) + try: + req = init_requirement(line) + except Exception: + raise RequirementError( + "Supplied requirement is not installable: {0!r}".format(line) + ) + else: + name = getattr(req, "name", None) + line = getattr(req, "url", None) vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line(line) setup_path = Path(path) / "setup.py" if path else None arg_dict = { @@ -389,8 +421,12 @@ class FileRequirement(BaseRequirement): from pip_shims import Wheel arg_dict["name"] = Wheel(link.filename).name + elif name: + arg_dict["name"] = name elif link.egg_fragment: arg_dict["name"] = link.egg_fragment + if req: + arg_dict["req"] = req created = cls(**arg_dict) return created @@ -428,7 +464,9 @@ class FileRequirement(BaseRequirement): if not uri: uri = pip_shims.shims.path_to_url(path) link = create_link(uri) - + req = None + if link.is_artifact and not link.is_wheel and not link.scheme.startswith("file"): + req = init_requirement("{0}@{1}".format(name, uri)) arg_dict = { "name": name, "path": path, @@ -437,6 +475,8 @@ class FileRequirement(BaseRequirement): "link": link, "uri_scheme": uri_scheme, } + if req: + arg_dict["req"] = req return cls(**arg_dict) @property @@ -449,7 +489,10 @@ class FileRequirement(BaseRequirement): seed = unquote(self.link.url_without_fragment) or self.uri # add egg fragments to remote artifacts (valid urls only) if not self._has_hashed_name and self.is_remote_artifact: - seed += "#egg={0}".format(self.name) + if not self.link.is_wheel and self.link.is_artifact: + seed = "{0}@{1}".format(self.name, seed) + else: + seed += "#egg={0}".format(self.name) editable = "-e " if self.editable else "" return "{0}{1}".format(editable, seed) @@ -575,7 +618,8 @@ class VCSRequirement(FileRequirement): ) req = init_requirement(canonicalize_name(self.name)) req.editable = self.editable - req.url = self.uri + if not getattr(req, "url") and self.uri: + req.url = self.uri req.line = self.link.url if self.ref: req.revision = self.ref @@ -813,7 +857,7 @@ class VCSRequirement(FileRequirement): class Requirement(object): name = attr.ib() vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs)) - req = attr.ib(default=None, validator=optional_instance_of(BaseRequirement)) + req = attr.ib(default=None) markers = attr.ib(default=None) specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers)) index = attr.ib(default=None) @@ -915,8 +959,11 @@ class Requirement(object): # Installable local files and installable non-vcs urls are handled # as files, generally speaking line_is_vcs = is_vcs(line) + # check for pep-508 compatible requirements + name, _, possible_url = line.partition("@") if is_installable_file(line) or ( - (is_file_url(line) or is_valid_url(line)) and not line_is_vcs + (is_valid_url(possible_url) or is_file_url(line) or is_valid_url(line)) and + not (line_is_vcs or is_vcs(possible_url)) ): r = FileRequirement.from_line(line_with_prefix) elif line_is_vcs: diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index fbaaf1a4..aa7ffd68 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -424,17 +424,18 @@ def make_install_requirement(name, version, extras, markers, constraint=False): """ # If no extras are specified, the extras string is blank + from pip_shims.shims import install_req_from_line extras_string = "" if extras: # Sort extras for stability extras_string = "[{}]".format(",".join(sorted(extras))) if not markers: - return ireq_from_line( + return install_req_from_line( str('{}{}=={}'.format(name, extras_string, version)), constraint=constraint) else: - return ireq_from_line( + return install_req_from_line( str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), constraint=constraint) diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index f8c4e009..f0d9e77f 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -153,6 +153,7 @@ class VistirSpinner(base_obj): def ok(self, text="OK"): """Set Ok (success) finalizer to a spinner.""" + # Do not display spin text for ok state self._text = None _text = text if text else "OK" @@ -160,6 +161,7 @@ class VistirSpinner(base_obj): def fail(self, text="FAIL"): """Set fail finalizer to a spinner.""" + # Do not display spin text for fail state self._text = None _text = text if text else "FAIL" From 808fd344426c526f790ccfc1f1c93c535f57da5f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 8 Nov 2018 04:49:52 -0500 Subject: [PATCH 125/218] Less checkmarks during lock Signed-off-by: Dan Ryan --- pipenv/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 93026ca7..8674aee2 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -579,9 +579,6 @@ def venv_resolve_deps( dev=dev, ) vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] - sp.write(environments.PIPENV_SPINNER_OK_TEXT.format( - "Successfully pinned VCS Packages!" - )) cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() From 65d70905ee8d8e79bbe8ba8c5d5b72573f37e201 Mon Sep 17 00:00:00 2001 From: frostming Date: Thu, 8 Nov 2018 17:52:05 +0800 Subject: [PATCH 126/218] Force inline table --- pipenv/project.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/project.py b/pipenv/project.py index 6d59b4cb..4f947e4b 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -580,7 +580,7 @@ class Project(object): @staticmethod def convert_outline_table(parsed): """Converts all outline to inline tables""" - if Project._istomlkit_parsed_result(parsed): + if Project._is_tomlkit_parsed_result(parsed): empty_inline_table = tomlkit.inline_table else: empty_inline_table = toml.TomlDecoder().get_empty_inline_table @@ -856,6 +856,7 @@ class Project(object): """Writes the given data structure out as TOML.""" if path is None: path = self.pipfile_location + data = self.convert_outline_table(data) try: formatted_data = tomlkit.dumps(data).rstrip() except Exception: From 5b496705a095db7cbb89065fce0b90fe8e9e3f42 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 8 Nov 2018 13:13:54 -0500 Subject: [PATCH 127/218] Fix broken requirementslib updates Signed-off-by: Dan Ryan --- .../requirementslib/models/requirements.py | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index ce2b0927..a36e5ba4 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -143,6 +143,7 @@ class FileRequirement(object): editable = attr.ib(default=False) #: Extras if applicable extras = attr.ib(default=attr.Factory(list)) + _uri_scheme = attr.ib(default=None) #: URI of the package uri = attr.ib() #: Link object representing the package to clone @@ -154,7 +155,6 @@ class FileRequirement(object): req = attr.ib() #: Whether this is a direct url requirement is_direct = attr.ib(default=False) - _uri_scheme = attr.ib(default=None) @classmethod def get_link_from_line(cls, line): @@ -276,7 +276,7 @@ class FileRequirement(object): if loc: self._uri_scheme = "path" if self.path else "uri" name = None - if self.req and getattr(self.req, "name"): + if getattr(self, "req", None) and getattr(self.req, "name"): return self.req.name if self.link and self.link.egg_fragment: return self.link.egg_fragment @@ -339,25 +339,27 @@ class FileRequirement(object): if self.link.is_artifact and not self.editable: if self._uri_scheme == "uri": if self.name: - req_str = "{0} @{1}".format(self.name, self.link.url_without_fragment) + req_str = "{0} @ {1}".format(self.name, self.link.url_without_fragment) else: req_str = "{0}".format(self.link.url_without_fragment) req = init_requirement(req_str) req.line = req_str + else: + req = init_requirement(normalize_name(self.name)) else: req = init_requirement(normalize_name(self.name)) req.editable = False req.line = self.link.url_without_fragment - if self.path and self.link and self.link.scheme.startswith("file"): - req.local_file = True - req.path = self.path - req.url = None - self._uri_scheme = "file" - else: - req.local_file = False - req.path = None - if not getattr(req, "url", None): - req.url = self.link.url_without_fragment + if self.path and self.link and self.link.scheme.startswith("file"): + req.local_file = True + req.path = self.path + req.url = None + self._uri_scheme = "file" + else: + req.local_file = False + req.path = None + if not getattr(req, "url", None): + req.url = self.link.url_without_fragment if self.editable: req.editable = True req.link = self.link @@ -1148,7 +1150,7 @@ class Requirement(object): req.line = req_line req.specifier = SpecifierSet(self.specifiers if self.specifiers else "") if self.is_vcs or self.is_file_or_url: - req.url = self.req.link.url_without_fragment + req.url = getattr(self.req.req, "url", self.req.link.url_without_fragment) req.marker = self.get_markers() req.extras = set(self.extras) if self.extras else set() return req From 60947a07eddec928cd23ca9e1c2d443b4a59d93f Mon Sep 17 00:00:00 2001 From: jxltom Date: Fri, 9 Nov 2018 09:50:01 +0800 Subject: [PATCH 128/218] Remove blank lines in pipenv graph --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index dabf065c..83b9a2a7 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2517,7 +2517,7 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): click.echo(simplejson.dumps(data, indent=4)) sys.exit(0) else: - for line in c.out.split("\n"): + for line in c.out.strip().split("\n"): # Ignore bad packages as top level. if line.split("==")[0] in BAD_PACKAGES and not reverse: continue From fc2c2ef29ee982906cc8f3a402286ffbd1370c0d Mon Sep 17 00:00:00 2001 From: jxltom Date: Fri, 9 Nov 2018 09:51:20 +0800 Subject: [PATCH 129/218] Add traivial news for issue 3199 --- news/3199.trivial | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/3199.trivial diff --git a/news/3199.trivial b/news/3199.trivial new file mode 100644 index 00000000..1fcca7de --- /dev/null +++ b/news/3199.trivial @@ -0,0 +1 @@ +Remove blank lines for pipenv graph From dbf21d5195e82e6d03816e92dd33b37c3c0577ca Mon Sep 17 00:00:00 2001 From: jxltom Date: Fri, 9 Nov 2018 09:56:04 +0800 Subject: [PATCH 130/218] Do not raise exception for pipenv graph when virtualenv does not exist --- pipenv/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pipenv/core.py b/pipenv/core.py index dabf065c..d3d05a0d 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2438,6 +2438,9 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): err=True, ) sys.exit(1) + except RuntimeError: + pass + if reverse and json: click.echo( u"{0}: {1}".format( From 9d33d6323dc69199ed4ce536f14177ee963fac5e Mon Sep 17 00:00:00 2001 From: jxltom Date: Fri, 9 Nov 2018 10:00:29 +0800 Subject: [PATCH 131/218] Add bugfix news for issue 3201 --- news/3201.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/3201.bugfix diff --git a/news/3201.bugfix b/news/3201.bugfix new file mode 100644 index 00000000..01dba4f6 --- /dev/null +++ b/news/3201.bugfix @@ -0,0 +1 @@ +Fix a bug where exception is raised when run pipenv graph in a project without created virtualenv From 3490fc85efc652f9f20dedd2c3dc6f5d1a2087e1 Mon Sep 17 00:00:00 2001 From: frostming Date: Fri, 9 Nov 2018 11:23:02 +0800 Subject: [PATCH 132/218] fix patch --- pipenv/vendor/tomlkit/container.py | 3 +-- .../patches/vendor/tomlkit-update-items.patch | 11 +++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index 5ddd72e7..987a0790 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -196,8 +196,7 @@ class Container(dict): self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) else: self._body[idx] = (None, Null()) - super(Container, self).__delitem__(key.key) - + super(Container, self).__delitem__(key.key) return self diff --git a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch index 51f6006a..c996cb96 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch @@ -9,11 +9,11 @@ index 37014921..5ddd72e7 100644 +from .items import Trivia from .items import Whitespace from .items import item as _item - + @@ -189,9 +190,14 @@ class Container(dict): if idx is None: raise NonExistentKey(key) - + - self._body[idx] = (None, Null()) + old_data = self._body[idx][1] + trivia = getattr(old_data, "trivia", None) @@ -21,9 +21,8 @@ index 37014921..5ddd72e7 100644 + self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) + else: + self._body[idx] = (None, Null()) -+ super(Container, self).__delitem__(key.key) - ++ super(Container, self).__delitem__(key.key) + - super(Container, self).__delitem__(key.key) - + return self - From a067a084789ed9c0506bbbb6d5904459a3b1655c Mon Sep 17 00:00:00 2001 From: jxltom Date: Fri, 9 Nov 2018 11:35:11 +0800 Subject: [PATCH 133/218] Pass requirements para to only in get_requirements --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index dabf065c..c77e14a1 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -721,7 +721,7 @@ def do_install_dependencies( # Allow pip to resolve dependencies when in skip-lock mode. no_deps = not skip_lock failed_deps_list = [] - deps_list = list(lockfile.get_requirements(dev=dev, only=True)) + deps_list = list(lockfile.get_requirements(dev=dev, only=requirements)) if requirements: index_args = prepare_pip_source_args(project.sources) index_args = " ".join(index_args).replace(" -", "\n-") From de78c1efce2da437ed39f014509f576305bb0f2f Mon Sep 17 00:00:00 2001 From: frostming Date: Fri, 9 Nov 2018 12:59:51 +0800 Subject: [PATCH 134/218] Only convert outline tables when write toml --- pipenv/project.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index f58b9c58..bed430af 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -585,25 +585,28 @@ class Project(object): else: empty_inline_table = toml.TomlDecoder().get_empty_inline_table for section in ("packages", "dev-packages"): + has_outline_table = False table_data = parsed.get(section, {}).copy() for package, value in table_data.items(): - if hasattr(value, "keys"): + if hasattr(value, "keys") and not isinstance( + value, (tomlkit.items.InlineTable, toml.decoder.InlineTableDict) + ): + has_outline_table = True table = empty_inline_table() table.update(value) table_data[package] = table - parsed[section] = table_data + if has_outline_table: + # We'll lose comments here, only update when necessary + parsed[section] = table_data return parsed def _parse_pipfile(self, contents): try: - data = tomlkit.parse(contents) + return tomlkit.parse(contents) except Exception: # We lose comments here, but it's for the best.) # Fallback to toml parser, for large files. - data = toml.loads(contents) - if "[packages." in contents or "[dev-packages." in contents: - data = self.convert_outline_table(data) - return data + return toml.loads(contents) def _read_pyproject(self): pyproject = self.path_to("pyproject.toml") From accd0ea4abdb1cbe35c094e01cc93b36147c44a3 Mon Sep 17 00:00:00 2001 From: frostming Date: Fri, 9 Nov 2018 14:18:04 +0800 Subject: [PATCH 135/218] Move to utils function --- pipenv/project.py | 31 ++----------------- pipenv/utils.py | 25 +++++++++++++++ .../patches/vendor/tomlkit-update-items.patch | 15 +++++---- 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index bed430af..04b519ed 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -25,6 +25,7 @@ from .utils import ( find_requirements, is_editable, cleanup_toml, + convert_toml_outline_tables, is_installable_file, is_valid_url, normalize_drive, @@ -572,34 +573,6 @@ class Project(object): """Clear pipfile cache (e.g., so we can mutate parsed pipfile)""" _pipfile_cache.clear() - @staticmethod - def _is_tomlkit_parsed_result(parsed): - """Check by duck typing of tomlkit.api.Container""" - return hasattr(parsed, "_body") - - @staticmethod - def convert_outline_table(parsed): - """Converts all outline to inline tables""" - if Project._is_tomlkit_parsed_result(parsed): - empty_inline_table = tomlkit.inline_table - else: - empty_inline_table = toml.TomlDecoder().get_empty_inline_table - for section in ("packages", "dev-packages"): - has_outline_table = False - table_data = parsed.get(section, {}).copy() - for package, value in table_data.items(): - if hasattr(value, "keys") and not isinstance( - value, (tomlkit.items.InlineTable, toml.decoder.InlineTableDict) - ): - has_outline_table = True - table = empty_inline_table() - table.update(value) - table_data[package] = table - if has_outline_table: - # We'll lose comments here, only update when necessary - parsed[section] = table_data - return parsed - def _parse_pipfile(self, contents): try: return tomlkit.parse(contents) @@ -860,7 +833,7 @@ class Project(object): """Writes the given data structure out as TOML.""" if path is None: path = self.pipfile_location - data = self.convert_outline_table(data) + data = convert_toml_outline_tables(data) try: formatted_data = tomlkit.dumps(data).rstrip() except Exception: diff --git a/pipenv/utils.py b/pipenv/utils.py index 8674aee2..9d495fa3 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -7,6 +7,8 @@ import re import shutil import stat import sys +import toml +import tomlkit import warnings import crayons @@ -93,6 +95,29 @@ def cleanup_toml(tml): return toml +def convert_toml_outline_tables(parsed): + """Converts all outline tables to inline tables.""" + if isinstance(parsed, tomlkit.container.Container): + empty_inline_table = tomlkit.inline_table + else: + empty_inline_table = toml.TomlDecoder().get_empty_inline_table + for section in ("packages", "dev-packages"): + has_outline_table = False + table_data = parsed.get(section, {}).copy() + for package, value in table_data.items(): + if hasattr(value, "keys") and not isinstance( + value, (tomlkit.items.InlineTable, toml.decoder.InlineTableDict) + ): + has_outline_table = True + table = empty_inline_table() + table.update(value) + table_data[package] = table + if has_outline_table: + # We'll lose comments here, only update when necessary + parsed[section] = table_data + return parsed + + def parse_python_version(output): """Parse a Python version output returned by `python --version`. diff --git a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch index c996cb96..ed2fb95e 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py -index 37014921..5ddd72e7 100644 +index 37014921..987a0790 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -9,6 +9,7 @@ from .items import Item @@ -9,20 +9,19 @@ index 37014921..5ddd72e7 100644 +from .items import Trivia from .items import Whitespace from .items import item as _item - -@@ -189,9 +190,14 @@ class Container(dict): + +@@ -189,8 +190,12 @@ class Container(dict): if idx is None: raise NonExistentKey(key) - + - self._body[idx] = (None, Null()) +- + old_data = self._body[idx][1] + trivia = getattr(old_data, "trivia", None) + if trivia and getattr(trivia, "comment", None): + self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) + else: + self._body[idx] = (None, Null()) -+ super(Container, self).__delitem__(key.key) - -- super(Container, self).__delitem__(key.key) - + super(Container, self).__delitem__(key.key) + return self From 2e90a6fd9859c1466629af1cbc9f3d7521163753 Mon Sep 17 00:00:00 2001 From: David Staheli Date: Fri, 9 Nov 2018 15:56:30 -0500 Subject: [PATCH 136/218] Remove old .vsts-ci directory and its contents --- .vsts-ci/docs.yml | 20 ---------- .vsts-ci/linux.yml | 27 -------------- .vsts-ci/phases/run-manifest-check.yml | 14 ------- .vsts-ci/phases/run-tests-windows.yml | 12 ------ .vsts-ci/phases/run-tests.yml | 37 ------------------- .vsts-ci/phases/run-vendor-scripts.yml | 39 -------------------- .vsts-ci/phases/test.yml | 49 ------------------------- .vsts-ci/steps/create-virtualenv.yml | 6 --- .vsts-ci/steps/install-dependencies.yml | 3 -- .vsts-ci/steps/run-tests.yml | 23 ------------ .vsts-ci/windows.yml | 23 ------------ 11 files changed, 253 deletions(-) delete mode 100644 .vsts-ci/docs.yml delete mode 100644 .vsts-ci/linux.yml delete mode 100644 .vsts-ci/phases/run-manifest-check.yml delete mode 100644 .vsts-ci/phases/run-tests-windows.yml delete mode 100644 .vsts-ci/phases/run-tests.yml delete mode 100644 .vsts-ci/phases/run-vendor-scripts.yml delete mode 100644 .vsts-ci/phases/test.yml delete mode 100644 .vsts-ci/steps/create-virtualenv.yml delete mode 100644 .vsts-ci/steps/install-dependencies.yml delete mode 100644 .vsts-ci/steps/run-tests.yml delete mode 100644 .vsts-ci/windows.yml diff --git a/.vsts-ci/docs.yml b/.vsts-ci/docs.yml deleted file mode 100644 index 1b9d3701..00000000 --- a/.vsts-ci/docs.yml +++ /dev/null @@ -1,20 +0,0 @@ -phases: -- phase: - displayName: Docs - queue: Hosted Linux Preview - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - - - template: steps/install-dependencies.yml - - - bash: tox -e docs - displayName: Build docs - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: docs' - inputs: - PathtoPublish: docs/build - ArtifactName: docs diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml deleted file mode 100644 index 95c62dc5..00000000 --- a/.vsts-ci/linux.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Pipenv Build Rules -trigger: - batch: true - branches: - include: - - master - paths: - exclude: - - docs/* - - news/* - - README.md - - pipenv/*.txt - - CHANGELOG.rst - - CONTRIBUTING.md - - CODE_OF_CONDUCT.md - - .gitignore - - .gitattributes - - .editorconfig - -phases: -- template: phases/test.yml - parameters: - queue: Hosted Linux Preview - -- template: phases/run-vendor-scripts.yml - parameters: - queue: Hosted Linux Preview diff --git a/.vsts-ci/phases/run-manifest-check.yml b/.vsts-ci/phases/run-manifest-check.yml deleted file mode 100644 index 484641dc..00000000 --- a/.vsts-ci/phases/run-manifest-check.yml +++ /dev/null @@ -1,14 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - -- template: ../steps/install-dependencies.yml - -- bash: | - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 - python -m pip install check-manifest - check-manifest diff --git a/.vsts-ci/phases/run-tests-windows.yml b/.vsts-ci/phases/run-tests-windows.yml deleted file mode 100644 index 6b6f86fa..00000000 --- a/.vsts-ci/phases/run-tests-windows.yml +++ /dev/null @@ -1,12 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - -- template: ../steps/install-dependencies.yml - -- template: ../steps/create-virtualenv.yml - -- template: ../steps/run-tests.yml diff --git a/.vsts-ci/phases/run-tests.yml b/.vsts-ci/phases/run-tests.yml deleted file mode 100644 index fe35ad8d..00000000 --- a/.vsts-ci/phases/run-tests.yml +++ /dev/null @@ -1,37 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - -- template: ../steps/install-dependencies.yml - -- bash: | - mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" - mkdir -p "$WORKON_HOME" - pip install certifi - export GIT_SSL_CAINFO="$(python -m certifi)" - export LANG="C.UTF-8" - export PIP_PROCESS_DEPENDENCY_LINKS="1" - echo "Path $PATH" - echo "Installing Pipenv…" - pip install -e "$(pwd)" --upgrade - pipenv install --deploy --dev - echo pipenv --venv && echo pipenv --py && echo pipenv run python --version - displayName: Make Virtualenv - -- script: | - # Fix Git SSL errors - export GIT_SSL_CAINFO="$(python -m certifi)" - export LANG="C.UTF-8" - export PIP_PROCESS_DEPENDENCY_LINKS="1" - pipenv run pytest --junitxml=test-results.xml - displayName: Run integration tests - -- task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.vsts-ci/phases/run-vendor-scripts.yml b/.vsts-ci/phases/run-vendor-scripts.yml deleted file mode 100644 index b96f64f5..00000000 --- a/.vsts-ci/phases/run-vendor-scripts.yml +++ /dev/null @@ -1,39 +0,0 @@ -parameters: - queue: - -phases: -- phase: Vendor_Scripts - displayName: Test Vendor Scripts - - queue: - name: ${{ parameters.queue }} - parallel: 4 - matrix: - ${{ if eq(parameters.queue, 'Hosted VS2017') }}: - # TODO remove once Hosted VS2017 has Python 3.7 - Python37: - python.version: '>= 3.7.0-b2' - python.architecture: x64 - ${{ if ne(parameters.queue, 'Hosted VS2017' )}}: - Python37: - python.version: '>= 3.7' - python.architecture: x64 - steps: - - task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - - - template: ../steps/install-dependencies.yml - - - bash: | - mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" - mkdir -p "$WORKON_HOME" - pip install certifi - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 - python -m pip install --upgrade invoke requests parver - python -m invoke vendoring.update - - - template: ./run-manifest-check.yml diff --git a/.vsts-ci/phases/test.yml b/.vsts-ci/phases/test.yml deleted file mode 100644 index b0de1e23..00000000 --- a/.vsts-ci/phases/test.yml +++ /dev/null @@ -1,49 +0,0 @@ -parameters: - queue: - -phases: -- phase: Test_Primary - displayName: Test Primary - - queue: - name: ${{ parameters.queue }} - parallel: 4 - matrix: - Python27: - python.version: '2.7' - python.architecture: x64 - ${{ if eq(parameters.queue, 'Hosted VS2017') }}: - # TODO remove once Hosted VS2017 has Python 3.7 - Python37: - python.version: '>= 3.7.0-b2' - python.architecture: x64 - ${{ if ne(parameters.queue, 'Hosted VS2017' )}}: - Python37: - python.version: '>= 3.7' - python.architecture: x64 - steps: - - ${{ if eq(parameters.queue, 'Hosted VS2017') }}: - - template: ./run-tests-windows.yml - - - ${{ if ne(parameters.queue, 'Hosted VS2017') }}: - - template: ./run-tests.yml - -- phase: Test_Secondary - displayName: Test python3.6 - # Run after Test_Primary so we don't devour time and jobs if tests are going to fail -# dependsOn: Test_Primary - - queue: - name: ${{ parameters.queue }} - parallel: 4 - matrix: - Python36: - python.version: '3.6' - python.architecture: x64 - steps: - - ${{ if eq(parameters.queue, 'Hosted VS2017') }}: - - template: ./run-tests-windows.yml - - - ${{ if ne(parameters.queue, 'Hosted VS2017') }}: - - template: ./run-tests.yml - diff --git a/.vsts-ci/steps/create-virtualenv.yml b/.vsts-ci/steps/create-virtualenv.yml deleted file mode 100644 index a72e9964..00000000 --- a/.vsts-ci/steps/create-virtualenv.yml +++ /dev/null @@ -1,6 +0,0 @@ -steps: -- script: | - virtualenv D:\.venv - D:\.venv\Scripts\pip.exe install -e . && D:\.venv\Scripts\pipenv install --dev - echo D:\.venv\Scripts\pipenv --venv && echo D:\.venv\Scripts\pipenv --py && echo D:\.venv\Scripts\pipenv run python --version - displayName: Make Virtualenv diff --git a/.vsts-ci/steps/install-dependencies.yml b/.vsts-ci/steps/install-dependencies.yml deleted file mode 100644 index dfe733b6..00000000 --- a/.vsts-ci/steps/install-dependencies.yml +++ /dev/null @@ -1,3 +0,0 @@ -steps: -- script: 'python -m pip install --upgrade pip && python -m pip install -e .' - displayName: Upgrade Pip & Install Pipenv diff --git a/.vsts-ci/steps/run-tests.yml b/.vsts-ci/steps/run-tests.yml deleted file mode 100644 index a7b99a13..00000000 --- a/.vsts-ci/steps/run-tests.yml +++ /dev/null @@ -1,23 +0,0 @@ -steps: -- powershell: | - # Fix Git SSL errors - pip install certifi - python -m certifi > cacert.txt - Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" - $env:GIT_SSL_CAINFO="$(Get-Content cacert.txt)" - # Shorten paths to get under MAX_PATH or else integration tests will fail - # https://bugs.python.org/issue18199 - subst T: "$env:TEMP" - Write-Host "##vso[task.setvariable variable=TEMP]T:\" - $env:TEMP='T:\' - Write-Host "##vso[task.setvariable variable=TMP]T:\" - $env:TEMP='T:\' - D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests - displayName: Run integration tests - -- task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml deleted file mode 100644 index e423c3ab..00000000 --- a/.vsts-ci/windows.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Pipenv Build Rules -trigger: - batch: true - branches: - include: - - master - paths: - exclude: - - docs/* - - news/* - - README.md - - pipenv/*.txt - - CHANGELOG.rst - - CONTRIBUTING.md - - CODE_OF_CONDUCT.md - - .gitignore - - .gitattributes - - .editorconfig - -phases: -- template: phases/test.yml - parameters: - queue: Hosted VS2017 From 6df7d8861da841e552049dcde9ff9a0f23edc01e Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 10 Nov 2018 12:34:54 +0800 Subject: [PATCH 137/218] update tomlkit --- Pipfile.lock | 6 +- pipenv/vendor/tomlkit/__init__.py | 2 +- pipenv/vendor/tomlkit/_compat.py | 2 + pipenv/vendor/tomlkit/_utils.py | 17 +- pipenv/vendor/tomlkit/api.py | 2 + pipenv/vendor/tomlkit/container.py | 129 ++++-- pipenv/vendor/tomlkit/exceptions.py | 70 +++- pipenv/vendor/tomlkit/items.py | 53 ++- pipenv/vendor/tomlkit/parser.py | 592 +++++++++++++++------------- pipenv/vendor/tomlkit/source.py | 195 +++++++++ pipenv/vendor/tomlkit/toml_char.py | 19 +- pipenv/vendor/tomlkit/toml_file.py | 3 + pipenv/vendor/vendor.txt | 1 + 13 files changed, 745 insertions(+), 346 deletions(-) create mode 100644 pipenv/vendor/tomlkit/source.py diff --git a/Pipfile.lock b/Pipfile.lock index 3990a451..30a618ac 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -626,10 +626,10 @@ }, "tomlkit": { "hashes": [ - "sha256:8ab16e93162fc44d3ad83d2aa29a7140b8f7d996ae1790a73b9a7aed6fb504ac", - "sha256:ca181cee7aee805d455628f7c94eb8ae814763769a93e69157f250fe4ebe1926" + "sha256:82a8fbb8d8c6af72e96ba00b9db3e20ef61be6c79082552c9363f4559702258b", + "sha256:a43e0195edc9b3c198cd4b5f0f3d427a395d47c4a76ceba7cc875ed030756c39" ], - "version": "==0.4.4" + "version": "==0.5.2" }, "towncrier": { "editable": true, diff --git a/pipenv/vendor/tomlkit/__init__.py b/pipenv/vendor/tomlkit/__init__.py index 89e4cf59..92bfa27c 100644 --- a/pipenv/vendor/tomlkit/__init__.py +++ b/pipenv/vendor/tomlkit/__init__.py @@ -22,4 +22,4 @@ from .api import value from .api import ws -__version__ = "0.4.6" +__version__ = "0.5.2" diff --git a/pipenv/vendor/tomlkit/_compat.py b/pipenv/vendor/tomlkit/_compat.py index f94bb10e..b7407af6 100644 --- a/pipenv/vendor/tomlkit/_compat.py +++ b/pipenv/vendor/tomlkit/_compat.py @@ -141,9 +141,11 @@ PY36 = sys.version_info >= (3, 6) if PY2: unicode = unicode chr = unichr + long = long else: unicode = str chr = chr + long = int def decode(string, encodings=None): diff --git a/pipenv/vendor/tomlkit/_utils.py b/pipenv/vendor/tomlkit/_utils.py index f62a354a..0a68be9f 100644 --- a/pipenv/vendor/tomlkit/_utils.py +++ b/pipenv/vendor/tomlkit/_utils.py @@ -9,19 +9,30 @@ from datetime import timedelta from ._compat import decode from ._compat import timezone +RFC_3339_LOOSE = re.compile( + "^" + r"(([0-9]+)-(\d{2})-(\d{2}))?" # Date + "(" + "([T ])?" # Separator + r"(\d{2}):(\d{2}):(\d{2})(\.([0-9]+))?" # Time + r"((Z)|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone + ")?" + "$" +) + RFC_3339_DATETIME = re.compile( "^" "([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])" # Date "[T ]" # Separator - "([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?" # Time - "((Z)|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone + r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?" # Time + r"((Z)|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone "$" ) RFC_3339_DATE = re.compile("^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$") RFC_3339_TIME = re.compile( - "^([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?$" + r"^([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?$" ) _utc = timezone(timedelta(), "UTC") diff --git a/pipenv/vendor/tomlkit/api.py b/pipenv/vendor/tomlkit/api.py index 0ac26752..e541c20c 100644 --- a/pipenv/vendor/tomlkit/api.py +++ b/pipenv/vendor/tomlkit/api.py @@ -1,5 +1,7 @@ import datetime as _datetime +from typing import Tuple + from ._utils import parse_rfc3339 from .container import Container from .items import AoT diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index 987a0790..cb8af1d5 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -1,5 +1,13 @@ from __future__ import unicode_literals +from typing import Any +from typing import Dict +from typing import Generator +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union + from ._compat import decode from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey @@ -9,7 +17,6 @@ from .items import Item from .items import Key from .items import Null from .items import Table -from .items import Trivia from .items import Whitespace from .items import item as _item @@ -74,7 +81,7 @@ class Container(dict): return self.append(key, item) - def append(self, key, item): # type: (Union[Key, str], Item) -> Container + def append(self, key, item): # type: (Union[Key, str, None], Item) -> Container if not isinstance(key, Key) and key is not None: key = Key(key) @@ -99,7 +106,11 @@ class Container(dict): self.append(None, Whitespace("\n")) if key is not None and key in self: - current = self._body[self._map[key]][1] + current_idx = self._map[key] + if isinstance(current_idx, tuple): + current_idx = current_idx[0] + + current = self._body[current_idx][1] if isinstance(item, Table): if not isinstance(current, (Table, AoT)): raise KeyAlreadyPresent(key) @@ -121,7 +132,7 @@ class Container(dict): current.append(k, v) return self - else: + elif not item.is_super_table(): raise KeyAlreadyPresent(key) elif isinstance(item, AoT): if not isinstance(current, AoT): @@ -173,7 +184,23 @@ class Container(dict): else: return self._insert_at(0, key, item) - self._map[key] = len(self._body) + if key in self._map: + current_idx = self._map[key] + if isinstance(current_idx, tuple): + current_idx = current_idx[0] + + current = self._body[current_idx][1] + if key is not None and not isinstance(current, Table): + raise KeyAlreadyPresent(key) + + # Adding sub tables to a currently existing table + idx = self._map[key] + if not isinstance(idx, tuple): + idx = (idx,) + + self._map[key] = idx + (len(self._body),) + else: + self._map[key] = len(self._body) self._body.append((key, item)) @@ -190,12 +217,12 @@ class Container(dict): if idx is None: raise NonExistentKey(key) - old_data = self._body[idx][1] - trivia = getattr(old_data, "trivia", None) - if trivia and getattr(trivia, "comment", None): - self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) + if isinstance(idx, tuple): + for i in idx: + self._body[i] = (None, Null()) else: self._body[idx] = (None, Null()) + super(Container, self).__delitem__(key.key) return self @@ -224,7 +251,16 @@ class Container(dict): # Increment indices after the current index for k, v in self._map.items(): - if v > idx: + if isinstance(v, tuple): + new_indices = [] + for v_ in v: + if v_ > idx: + v_ = v_ + 1 + + new_indices.append(v_) + + self._map[k] = tuple(new_indices) + elif v > idx: self._map[k] = v + 1 self._map[other_key] = idx + 1 @@ -257,7 +293,16 @@ class Container(dict): # Increment indices after the current index for k, v in self._map.items(): - if v >= idx: + if isinstance(v, tuple): + new_indices = [] + for v_ in v: + if v_ >= idx: + v_ = v_ + 1 + + new_indices.append(v_) + + self._map[k] = tuple(new_indices) + elif v >= idx: self._map[k] = v + 1 self._map[key] = idx @@ -286,29 +331,7 @@ class Container(dict): s = "" for k, v in self._body: if k is not None: - if False: - key = k.as_string() - - for _k, _v in v.value.body: - if _k is None: - s += v.as_string() - elif isinstance(_v, Table): - s += v.as_string(prefix=key) - else: - _key = key - if prefix is not None: - _key = prefix + "." + _key - - s += "{}{}{}{}{}{}{}".format( - _v.trivia.indent, - _key + "." + decode(_k.as_string()), - _k.sep, - decode(_v.as_string()), - _v.trivia.comment_ws, - decode(_v.trivia.comment), - _v.trivia.trail, - ) - elif isinstance(v, Table): + if isinstance(v, Table): s += self._render_table(k, v) elif isinstance(v, AoT): s += self._render_aot(k, v) @@ -332,7 +355,12 @@ class Container(dict): if prefix is not None: _key = prefix + "." + _key - if not table.is_super_table(): + if not table.is_super_table() or ( + any( + not isinstance(v, (Table, AoT, Whitespace)) for _, v in table.value.body + ) + and not key.is_dotted() + ): open_, close = "[", "]" if table.is_aot_element(): open_, close = "[[", "]]" @@ -465,7 +493,7 @@ class Container(dict): return key in self._map - def __getitem__(self, key): # type: (Union[Key, str]) -> Item + def __getitem__(self, key): # type: (Union[Key, str]) -> Union[Item, Container] if not isinstance(key, Key): key = Key(key) @@ -473,6 +501,20 @@ class Container(dict): if idx is None: raise NonExistentKey(key) + if isinstance(idx, tuple): + container = Container(True) + + for i in idx: + item = self._body[i][1] + + if isinstance(item, Table): + for k, v in item.value.body: + container.append(k, v) + else: + container.append(key, item) + + return container + item = self._body[idx][1] return item.value @@ -503,11 +545,20 @@ class Container(dict): def _replace_at( self, idx, new_key, value - ): # type: (int, Union[Key, str], Item) -> None + ): # type: (Union[int, Tuple[int]], Union[Key, str], Item) -> None + if isinstance(idx, tuple): + for i in idx[1:]: + self._body[i] = (None, Null()) + + idx = idx[0] + k, v = self._body[idx] self._map[new_key] = self._map.pop(k) + if isinstance(self._map[new_key], tuple): + self._map[new_key] = self._map[new_key][0] + value = _item(value) # Copying trivia @@ -517,6 +568,10 @@ class Container(dict): value.trivia.comment = v.trivia.comment value.trivia.trail = v.trivia.trail + if isinstance(value, Table): + # Insert a cosmetic new line for tables + value.append(None, Whitespace("\n")) + self._body[idx] = (new_key, value) super(Container, self).__setitem__(new_key.key, value.value) diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py index 46ee938b..4fbc667b 100644 --- a/pipenv/vendor/tomlkit/exceptions.py +++ b/pipenv/vendor/tomlkit/exceptions.py @@ -1,3 +1,6 @@ +from typing import Optional + + class TOMLKitError(Exception): pass @@ -23,6 +26,14 @@ class ParseError(ValueError, TOMLKitError): "{} at line {} col {}".format(message, self._line, self._col) ) + @property + def line(self): + return self._line + + @property + def col(self): + return self._col + class MixedArrayTypesError(ParseError): """ @@ -35,6 +46,50 @@ class MixedArrayTypesError(ParseError): super(MixedArrayTypesError, self).__init__(line, col, message=message) +class InvalidNumberError(ParseError): + """ + A numeric field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid number" + + super(InvalidNumberError, self).__init__(line, col, message=message) + + +class InvalidDateTimeError(ParseError): + """ + A datetime field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid datetime" + + super(InvalidDateTimeError, self).__init__(line, col, message=message) + + +class InvalidDateError(ParseError): + """ + A date field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid date" + + super(InvalidDateError, self).__init__(line, col, message=message) + + +class InvalidTimeError(ParseError): + """ + A date field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid time" + + super(InvalidTimeError, self).__init__(line, col, message=message) + + class InvalidNumberOrDateError(ParseError): """ A numeric or date field was improperly specified. @@ -46,6 +101,17 @@ class InvalidNumberOrDateError(ParseError): super(InvalidNumberOrDateError, self).__init__(line, col, message=message) +class InvalidUnicodeValueError(ParseError): + """ + A unicode code was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid unicode value" + + super(InvalidUnicodeValueError, self).__init__(line, col, message=message) + + class UnexpectedCharError(ParseError): """ An unexpected character was found during parsing. @@ -106,7 +172,9 @@ class InternalParserError(ParseError): An error that indicates a bug in the parser. """ - def __init__(self, line, col, message=None): # type: (int, int) -> None + def __init__( + self, line, col, message=None + ): # type: (int, int, Optional[str]) -> None msg = "Internal parser error" if message: msg += " ({})".format(message) diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index c3c2d59f..375b5f02 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -6,14 +6,18 @@ import string from datetime import date from datetime import datetime from datetime import time -import sys -if sys.version_info >= (3, 4): - from enum import Enum -else: - from pipenv.vendor.backports.enum import Enum +from enum import Enum +from typing import Any +from typing import Dict +from typing import Generator +from typing import List +from typing import Optional +from typing import Union + from ._compat import PY2 from ._compat import decode +from ._compat import long from ._compat import unicode from ._utils import escape_string @@ -21,7 +25,6 @@ if PY2: from pipenv.vendor.backports.functools_lru_cache import lru_cache else: from functools import lru_cache -from toml.decoder import InlineTableDict def item(value, _parent=None): @@ -37,10 +40,7 @@ def item(value, _parent=None): elif isinstance(value, float): return Float(value, Trivia(), str(value)) elif isinstance(value, dict): - if isinstance(value, InlineTableDict): - val = InlineTable(Container(), Trivia()) - else: - val = Table(Container(), Trivia(), False) + val = Table(Container(), Trivia(), False) for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): val[k] = item(v, _parent=val) @@ -124,6 +124,24 @@ class StringType(Enum): }[self] +class BoolType(Enum): + TRUE = "true" + FALSE = "false" + + @lru_cache(maxsize=None) + def __bool__(self): + return {BoolType.TRUE: True, BoolType.FALSE: False}[self] + + if PY2: + __nonzero__ = __bool__ # for PY2 + + def __iter__(self): + return iter(self.value) + + def __len__(self): + return len(self.value) + + class Trivia: """ Trivia information (aka metadata). @@ -310,7 +328,7 @@ class Comment(Item): return "{}{}".format(self._trivia.indent, decode(self._trivia.comment)) -class Integer(int, Item): +class Integer(long, Item): """ An integer literal. """ @@ -449,10 +467,10 @@ class Bool(Item): A boolean literal. """ - def __init__(self, value, trivia): # type: (float, Trivia) -> None + def __init__(self, t, trivia): # type: (float, Trivia) -> None super(Bool, self).__init__(trivia) - self._value = value + self._value = bool(t) @property def discriminant(self): # type: () -> int @@ -747,10 +765,6 @@ class Table(Item, dict): def discriminant(self): # type: () -> int return 9 - @property - def value(self): # type: () -> tomlkit.container.Container - return self._value - def add(self, key, item=None): # type: (Union[Key, Item, str], Any) -> Item if item is None: if not isinstance(key, (Comment, Whitespace)): @@ -924,6 +938,8 @@ class InlineTable(Item, dict): if not isinstance(_item, (Whitespace, Comment)): if not _item.trivia.indent and len(self._value) > 0: _item.trivia.indent = " " + if _item.trivia.comment: + _item.trivia.comment = "" self._value.append(key, _item) @@ -1003,8 +1019,7 @@ class InlineTable(Item, dict): if key is not None: super(InlineTable, self).__setitem__(key, value) - - if hasattr(value, "trivia") and value.trivia.comment: + if value.trivia.comment: value.trivia.comment = "" m = re.match("(?s)^[^ ]*([ ]+).*$", self._trivia.indent) diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py index 7971d9a2..7b948331 100644 --- a/pipenv/vendor/tomlkit/parser.py +++ b/pipenv/vendor/tomlkit/parser.py @@ -1,24 +1,31 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import datetime -import itertools import re import string -from copy import copy +from typing import Any +from typing import Generator +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union -from ._compat import PY2 from ._compat import chr from ._compat import decode from ._utils import _escaped +from ._utils import RFC_3339_LOOSE from ._utils import parse_rfc3339 from .container import Container from .exceptions import EmptyKeyError from .exceptions import EmptyTableNameError from .exceptions import InternalParserError from .exceptions import InvalidCharInStringError -from .exceptions import InvalidNumberOrDateError +from .exceptions import InvalidDateTimeError +from .exceptions import InvalidDateError +from .exceptions import InvalidTimeError +from .exceptions import InvalidNumberError +from .exceptions import InvalidUnicodeValueError from .exceptions import MixedArrayTypesError from .exceptions import ParseError from .exceptions import UnexpectedCharError @@ -26,12 +33,14 @@ from .exceptions import UnexpectedEofError from .items import AoT from .items import Array from .items import Bool +from .items import BoolType from .items import Comment from .items import Date from .items import DateTime from .items import Float from .items import InlineTable from .items import Integer +from .items import Item from .items import Key from .items import KeyType from .items import Null @@ -41,6 +50,7 @@ from .items import Table from .items import Time from .items import Trivia from .items import Whitespace +from .source import Source from .toml_char import TOMLChar from .toml_document import TOMLDocument @@ -52,68 +62,69 @@ class Parser: def __init__(self, string): # type: (str) -> None # Input to parse - self._src = decode(string) # type: str - # Iterator used for getting characters from src. - self._chars = iter([(i, TOMLChar(c)) for i, c in enumerate(self._src)]) - # Current byte offset into src. - self._idx = 0 - # Current character - self._current = TOMLChar("") # type: TOMLChar - # Index into src between which and idx slices will be extracted - self._marker = 0 + self._src = Source(decode(string)) self._aot_stack = [] - self.inc() + @property + def _state(self): + return self._src.state + + @property + def _idx(self): + return self._src.idx + + @property + def _current(self): + return self._src.current + + @property + def _marker(self): + return self._src.marker def extract(self): # type: () -> str """ Extracts the value between marker and index """ - if self.end(): - return self._src[self._marker :] - else: - return self._src[self._marker : self._idx] + return self._src.extract() - def inc(self, exception=None): # type: () -> bool + def inc(self, exception=None): # type: (Optional[ParseError.__class__]) -> bool """ Increments the parser if the end of the input has not been reached. Returns whether or not it was able to advance. """ - try: - self._idx, self._current = next(self._chars) + return self._src.inc(exception=exception) - return True - except StopIteration: - self._idx = len(self._src) - self._current = TOMLChar("\0") - - if not exception: - return False - raise exception - - def inc_n(self, n, exception=None): # type: (int) -> bool + def inc_n(self, n, exception=None): # type: (int, Optional[ParseError]) -> bool """ Increments the parser by n characters if the end of the input has not been reached. """ - for _ in range(n): - if not self.inc(exception=exception): - return False + return self._src.inc_n(n=n, exception=exception) - return True + def consume(self, chars, min=0, max=-1): + """ + Consume chars until min/max is satisfied is valid. + """ + return self._src.consume(chars=chars, min=min, max=max) def end(self): # type: () -> bool """ Returns True if the parser has reached the end of the input. """ - return self._idx >= len(self._src) or self._current == "\0" + return self._src.end() def mark(self): # type: () -> None """ Sets the marker to the index's current position """ - self._marker = self._idx + self._src.mark() + + def parse_error(self, exception=ParseError, *args): + """ + Creates a generic "parse error" at the current position. + """ + return self._src.parse_error(exception, *args) def parse(self): # type: () -> TOMLDocument body = TOMLDocument(True) @@ -173,27 +184,6 @@ class Parser: return True - def parse_error(self, kind=ParseError, args=None): # type: () -> None - """ - Creates a generic "parse error" at the current position. - """ - line, col = self._to_linecol(self._idx) - - if args: - return kind(line, col, *args) - else: - return kind(line, col) - - def _to_linecol(self, offset): # type: (int) -> Tuple[int, int] - cur = 0 - for i, line in enumerate(self._src.splitlines()): - if cur + len(line) + 1 > offset: - return (i + 1, offset - cur) - - cur += len(line) + 1 - - return len(self._src.splitlines()), 0 - def _is_child(self, parent, child): # type: (str, str) -> bool """ Returns whether a key is strictly a child of another key. @@ -256,55 +246,35 @@ class Parser: if the item is value-like. """ self.mark() - saved_idx = self._save_idx() + with self._state as state: + while True: + c = self._current + if c == "\n": + # Found a newline; Return all whitespace found up to this point. + self.inc() - while True: - c = self._current - if c == "\n": - # Found a newline; Return all whitespace found up to this point. - self.inc() + return None, Whitespace(self.extract()) + elif c in " \t\r": + # Skip whitespace. + if not self.inc(): + return None, Whitespace(self.extract()) + elif c == "#": + # Found a comment, parse it + indent = self.extract() + cws, comment, trail = self._parse_comment_trail() - return (None, Whitespace(self.extract())) - elif c in " \t\r": - # Skip whitespace. - if not self.inc(): - return (None, Whitespace(self.extract())) - elif c == "#": - # Found a comment, parse it - indent = self.extract() - cws, comment, trail = self._parse_comment_trail() + return None, Comment(Trivia(indent, cws, comment, trail)) + elif c == "[": + # Found a table, delegate to the calling function. + return + else: + # Begining of a KV pair. + # Return to beginning of whitespace so it gets included + # as indentation for the KV about to be parsed. + state.restore = True + break - return (None, Comment(Trivia(indent, cws, comment, trail))) - elif c == "[": - # Found a table, delegate to the calling function. - return - else: - # Begining of a KV pair. - # Return to beginning of whitespace so it gets included - # as indentation for the KV about to be parsed. - self._restore_idx(*saved_idx) - key, value = self._parse_key_value(True) - - return key, value - - def _save_idx(self): # type: () -> Tuple[Iterator, int, str] - if PY2: - # Python 2.7 does not allow to directly copy - # an iterator, so we have to make tees of the original - # chars iterator. - chars1, chars2 = itertools.tee(self._chars) - - # We can no longer use the original chars iterator. - self._chars = chars1 - - return chars2, self._idx, self._current - - return copy(self._chars), self._idx, self._current - - def _restore_idx(self, chars, idx, current): # type: (Iterator, int, str) -> None - self._chars = chars - self._idx = idx - self._current = current + return self._parse_key_value(True) def _parse_comment_trail(self): # type: () -> Tuple[str, str, str] """ @@ -341,7 +311,7 @@ class Parser: elif c in " \t\r": self.inc() else: - raise self.parse_error(UnexpectedCharError, (c)) + raise self.parse_error(UnexpectedCharError, c) if self.end(): break @@ -361,9 +331,7 @@ class Parser: return comment_ws, comment, trail - def _parse_key_value( - self, parse_comment=False, inline=True - ): # type: (bool, bool) -> (Key, Item) + def _parse_key_value(self, parse_comment=False): # type: (bool) -> (Key, Item) # Leading indent self.mark() @@ -383,7 +351,7 @@ class Parser: while self._current.is_kv_sep() and self.inc(): if self._current == "=": if found_equals: - raise self.parse_error(UnexpectedCharError, ("=",)) + raise self.parse_error(UnexpectedCharError, "=") else: found_equals = True pass @@ -473,7 +441,7 @@ class Parser: def _handle_dotted_key( self, container, key, value - ): # type: (Container, Key) -> None + ): # type: (Container, Key, Any) -> None names = tuple(self._split_table_name(key.key)) name = names[0] name._dotted = True @@ -510,84 +478,22 @@ class Parser: Attempts to parse a value at the current position. """ self.mark() + c = self._current trivia = Trivia() - c = self._current - if c == '"': + if c == StringType.SLB.value: return self._parse_basic_string() - elif c == "'": + elif c == StringType.SLL.value: return self._parse_literal_string() - elif c == "t" and self._src[self._idx :].startswith("true"): - # Boolean: true - self.inc_n(4) - - return Bool(True, trivia) - elif c == "f" and self._src[self._idx :].startswith("false"): - # Boolean: true - self.inc_n(5) - - return Bool(False, trivia) + elif c == BoolType.TRUE.value[0]: + return self._parse_true() + elif c == BoolType.FALSE.value[0]: + return self._parse_false() elif c == "[": - # Array - elems = [] # type: List[Item] - self.inc() - - while self._current != "]": - self.mark() - while self._current.is_ws() or self._current == ",": - self.inc() - - if self._idx != self._marker: - elems.append(Whitespace(self.extract())) - - if self._current == "]": - break - - if self._current == "#": - cws, comment, trail = self._parse_comment_trail() - - next_ = Comment(Trivia("", cws, comment, trail)) - else: - next_ = self._parse_value() - - elems.append(next_) - - self.inc() - - try: - res = Array(elems, trivia) - except ValueError: - raise self.parse_error(MixedArrayTypesError) - - if res.is_homogeneous(): - return res - - raise self.parse_error(MixedArrayTypesError) + return self._parse_array() elif c == "{": - # Inline table - elems = Container(True) - self.inc() - - while self._current != "}": - self.mark() - while self._current.is_spaces() or self._current == ",": - self.inc() - - if self._idx != self._marker: - ws = self.extract().lstrip(",") - if ws: - elems.append(None, Whitespace(ws)) - - if self._current == "}": - break - - key, val = self._parse_key_value(False, inline=True) - elems.append(key, val) - - self.inc() - - return InlineTable(elems, trivia) - elif c in string.digits + "+-" or self._peek(4) in { + return self._parse_inline_table() + elif c in "+-" or self._peek(4) in { "+inf", "-inf", "inf", @@ -595,7 +501,7 @@ class Parser: "-nan", "nan", }: - # Integer, Float, Date, Time or DateTime + # Number while self._current not in " \t\n\r#,]}" and self.inc(): pass @@ -605,24 +511,166 @@ class Parser: if item is not None: return item - try: - res = parse_rfc3339(raw) - except ValueError: - res = None + raise self.parse_error(InvalidNumberError) + elif c in string.digits: + # Integer, Float, Date, Time or DateTime + while self._current not in " \t\n\r#,]}" and self.inc(): + pass - if res is None: - raise self.parse_error(InvalidNumberOrDateError) + raw = self.extract() - if isinstance(res, datetime.datetime): - return DateTime(res, trivia, raw) - elif isinstance(res, datetime.time): - return Time(res, trivia, raw) - elif isinstance(res, datetime.date): - return Date(res, trivia, raw) - else: - raise self.parse_error(InvalidNumberOrDateError) + m = RFC_3339_LOOSE.match(raw) + if m: + if m.group(1) and m.group(5): + # datetime + try: + return DateTime(parse_rfc3339(raw), trivia, raw) + except ValueError: + raise self.parse_error(InvalidDateTimeError) + + if m.group(1): + try: + return Date(parse_rfc3339(raw), trivia, raw) + except ValueError: + raise self.parse_error(InvalidDateError) + + if m.group(5): + try: + return Time(parse_rfc3339(raw), trivia, raw) + except ValueError: + raise self.parse_error(InvalidTimeError) + + item = self._parse_number(raw, trivia) + if item is not None: + return item + + raise self.parse_error(InvalidNumberError) else: - raise self.parse_error(UnexpectedCharError, (c)) + raise self.parse_error(UnexpectedCharError, c) + + def _parse_true(self): + return self._parse_bool(BoolType.TRUE) + + def _parse_false(self): + return self._parse_bool(BoolType.FALSE) + + def _parse_bool(self, style): # type: (BoolType) -> Bool + with self._state: + style = BoolType(style) + + # only keep parsing for bool if the characters match the style + # try consuming rest of chars in style + for c in style: + self.consume(c, min=1, max=1) + + return Bool(style, Trivia()) + + def _parse_array(self): # type: () -> Array + # Consume opening bracket, EOF here is an issue (middle of array) + self.inc(exception=UnexpectedEofError) + + elems = [] # type: List[Item] + prev_value = None + while True: + # consume whitespace + mark = self._idx + self.consume(TOMLChar.SPACES) + newline = self.consume(TOMLChar.NL) + indent = self._src[mark : self._idx] + if newline: + elems.append(Whitespace(indent)) + continue + + # consume comment + if self._current == "#": + cws, comment, trail = self._parse_comment_trail() + elems.append(Comment(Trivia(indent, cws, comment, trail))) + continue + + # consume indent + if indent: + elems.append(Whitespace(indent)) + continue + + # consume value + if not prev_value: + try: + elems.append(self._parse_value()) + prev_value = True + continue + except UnexpectedCharError: + pass + + # consume comma + if prev_value and self._current == ",": + self.inc(exception=UnexpectedEofError) + elems.append(Whitespace(",")) + prev_value = False + continue + + # consume closing bracket + if self._current == "]": + # consume closing bracket, EOF here doesn't matter + self.inc() + break + + raise self.parse_error(UnexpectedCharError, self._current) + + try: + res = Array(elems, Trivia()) + except ValueError: + pass + else: + if res.is_homogeneous(): + return res + + raise self.parse_error(MixedArrayTypesError) + + def _parse_inline_table(self): # type: () -> InlineTable + # consume opening bracket, EOF here is an issue (middle of array) + self.inc(exception=UnexpectedEofError) + + elems = Container(True) + trailing_comma = None + while True: + # consume leading whitespace + mark = self._idx + self.consume(TOMLChar.SPACES) + raw = self._src[mark : self._idx] + if raw: + elems.add(Whitespace(raw)) + + if not trailing_comma: + # None: empty inline table + # False: previous key-value pair was not followed by a comma + if self._current == "}": + # consume closing bracket, EOF here doesn't matter + self.inc() + break + if trailing_comma is False: + raise self.parse_error(UnexpectedCharError, self._current) + else: + # True: previous key-value pair was followed by a comma + if self._current == "}": + raise self.parse_error(UnexpectedCharError, self._current) + + key, val = self._parse_key_value(False) + elems.add(key, val) + + # consume trailing whitespace + mark = self._idx + self.consume(TOMLChar.SPACES) + raw = self._src[mark : self._idx] + if raw: + elems.add(Whitespace(raw)) + + # consume trailing comma + trailing_comma = self._current == "," + if trailing_comma: + # consume closing bracket, EOF here is an issue (middle of inline table) + self.inc(exception=UnexpectedEofError) + + return InlineTable(elems, Trivia()) def _parse_number(self, raw, trivia): # type: (str, Trivia) -> Optional[Item] # Leading zeros are not allowed @@ -670,11 +718,13 @@ class Parser: except ValueError: return - def _parse_literal_string(self): # type: () -> Item - return self._parse_string(StringType.SLL) + def _parse_literal_string(self): # type: () -> String + with self._state: + return self._parse_string(StringType.SLL) - def _parse_basic_string(self): # type: () -> Item - return self._parse_string(StringType.SLB) + def _parse_basic_string(self): # type: () -> String + with self._state: + return self._parse_string(StringType.SLB) def _parse_escaped_char(self, multiline): if multiline and self._current.is_ws(): @@ -696,7 +746,7 @@ class Parser: # the escape followed by whitespace must have a newline # before any other chars if "\n" not in tmp: - raise self.parse_error(InvalidCharInStringError, (self._current,)) + raise self.parse_error(InvalidCharInStringError, self._current) return "" @@ -717,15 +767,17 @@ class Parser: return u - raise self.parse_error(InvalidCharInStringError, (self._current,)) + raise self.parse_error(InvalidUnicodeValueError) - def _parse_string(self, delim): # type: (str) -> Item - delim = StringType(delim) - assert delim.is_singleline() + raise self.parse_error(InvalidCharInStringError, self._current) + def _parse_string(self, delim): # type: (StringType) -> String # only keep parsing for string if the current character matches the delim if self._current != delim.unit: - raise ValueError("Expecting a {!r} character".format(delim)) + raise self.parse_error( + InternalParserError, + "Invalid character for string type {}".format(delim), + ) # consume the opening/first delim, EOF here is an issue # (middle of string or middle of delim) @@ -755,7 +807,7 @@ class Parser: while True: if delim.is_singleline() and self._current.is_nl(): # single line cannot have actual newline characters - raise self.parse_error(InvalidCharInStringError, (self._current,)) + raise self.parse_error(InvalidCharInStringError, self._current) elif not escaped and self._current == delim.unit: # try to process current as a closing delim original = self.extract() @@ -781,8 +833,6 @@ class Parser: if not close: # if there is no close characters, keep parsing continue else: - close = delim.unit - # consume the closing delim, we do not care if EOF occurs as # that would simply imply the end of self._src self.inc() @@ -817,8 +867,7 @@ class Parser: """ if self._current != "[": raise self.parse_error( - InternalParserError, - ("_parse_table() called on non-bracket character.",), + InternalParserError, "_parse_table() called on non-bracket character." ) indent = self.extract() @@ -945,7 +994,7 @@ class Parser: else: raise self.parse_error( InternalParserError, - ("_parse_item() returned None on a non-bracket character.",), + "_parse_item() returned None on a non-bracket character.", ) if isinstance(result, Null): @@ -970,32 +1019,27 @@ class Parser: Returns the name of the table about to be parsed, as well as whether it is part of an AoT. """ - # Save initial state - idx = self._save_idx() - marker = self._marker + # we always want to restore after exiting this scope + with self._state(save_marker=True, restore=True): + if self._current != "[": + raise self.parse_error( + InternalParserError, + "_peek_table() entered on non-bracket character", + ) - if self._current != "[": - raise self.parse_error( - InternalParserError, ("_peek_table() entered on non-bracket character",) - ) - - # AoT - self.inc() - is_aot = False - if self._current == "[": + # AoT self.inc() - is_aot = True + is_aot = False + if self._current == "[": + self.inc() + is_aot = True - self.mark() + self.mark() - while self._current != "]" and self.inc(): - table_name = self.extract() + while self._current != "]" and self.inc(): + table_name = self.extract() - # Restore initial state - self._restore_idx(*idx) - self._marker = marker - - return is_aot, table_name + return is_aot, table_name def _parse_aot(self, first, name_first): # type: (Table, str) -> AoT """ @@ -1022,57 +1066,53 @@ class Parser: n is the max number of characters that will be peeked. """ - idx = self._save_idx() - buf = "" - for _ in range(n): - if self._current not in " \t\n\r#,]}": - buf += self._current - self.inc() - continue + # we always want to restore after exiting this scope + with self._state(restore=True): + buf = "" + for _ in range(n): + if self._current not in " \t\n\r#,]}": + buf += self._current + self.inc() + continue - break + break + return buf - self._restore_idx(*idx) - - return buf - - def _peek_unicode(self, is_long): # type: () -> Tuple[bool, str] + def _peek_unicode( + self, is_long + ): # type: (bool) -> Tuple[Optional[str], Optional[str]] """ Peeks ahead non-intrusively by cloning then restoring the initial state of the parser. Returns the unicode value is it's a valid one else None. """ - # Save initial state - idx = self._save_idx() - marker = self._marker + # we always want to restore after exiting this scope + with self._state(save_marker=True, restore=True): + if self._current not in {"u", "U"}: + raise self.parse_error( + InternalParserError, "_peek_unicode() entered on non-unicode value" + ) - if self._current not in {"u", "U"}: - raise self.parse_error( - InternalParserError, ("_peek_unicode() entered on non-unicode value") - ) + self.inc() # Dropping prefix + self.mark() - # AoT - self.inc() # Dropping prefix - self.mark() + if is_long: + chars = 8 + else: + chars = 4 - if is_long: - chars = 8 - else: - chars = 4 + if not self.inc_n(chars): + value, extracted = None, None + else: + extracted = self.extract() - if not self.inc_n(chars): - value, extracted = None, None - else: - extracted = self.extract() + if extracted[0].lower() == "d" and extracted[1].strip("01234567"): + return None, None - try: - value = chr(int(extracted, 16)) - except ValueError: - value = None + try: + value = chr(int(extracted, 16)) + except ValueError: + value = None - # Restore initial state - self._restore_idx(*idx) - self._marker = marker - - return value, extracted + return value, extracted diff --git a/pipenv/vendor/tomlkit/source.py b/pipenv/vendor/tomlkit/source.py new file mode 100644 index 00000000..1a96e058 --- /dev/null +++ b/pipenv/vendor/tomlkit/source.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import itertools + +from copy import copy +from typing import Optional +from typing import Tuple + +from ._compat import PY2 +from ._compat import unicode +from .exceptions import UnexpectedEofError +from .exceptions import UnexpectedCharError +from .exceptions import ParseError +from .toml_char import TOMLChar + + +class _State: + def __init__( + self, source, save_marker=False, restore=False + ): # type: (_Source, Optional[bool], Optional[bool]) -> None + self._source = source + self._save_marker = save_marker + self.restore = restore + + def __enter__(self): # type: () -> None + # Entering this context manager - save the state + if PY2: + # Python 2.7 does not allow to directly copy + # an iterator, so we have to make tees of the original + # chars iterator. + self._source._chars, self._chars = itertools.tee(self._source._chars) + else: + self._chars = copy(self._source._chars) + self._idx = self._source._idx + self._current = self._source._current + self._marker = self._source._marker + + return self + + def __exit__(self, exception_type, exception_val, trace): + # Exiting this context manager - restore the prior state + if self.restore or exception_type: + self._source._chars = self._chars + self._source._idx = self._idx + self._source._current = self._current + if self._save_marker: + self._source._marker = self._marker + + # Restore exceptions are silently consumed, other exceptions need to + # propagate + return exception_type is None + + +class _StateHandler: + """ + State preserver for the Parser. + """ + + def __init__(self, source): # type: (Source) -> None + self._source = source + self._states = [] + + def __call__(self, *args, **kwargs): + return _State(self._source, *args, **kwargs) + + def __enter__(self): # type: () -> None + state = self() + self._states.append(state) + return state.__enter__() + + def __exit__(self, exception_type, exception_val, trace): + state = self._states.pop() + return state.__exit__(exception_type, exception_val, trace) + + +class Source(unicode): + EOF = TOMLChar("\0") + + def __init__(self, _): # type: (unicode) -> None + super(Source, self).__init__() + + # Collection of TOMLChars + self._chars = iter([(i, TOMLChar(c)) for i, c in enumerate(self)]) + + self._idx = 0 + self._marker = 0 + self._current = TOMLChar("") + + self._state = _StateHandler(self) + + self.inc() + + def reset(self): + # initialize both idx and current + self.inc() + + # reset marker + self.mark() + + @property + def state(self): # type: () -> _StateHandler + return self._state + + @property + def idx(self): # type: () -> int + return self._idx + + @property + def current(self): # type: () -> TOMLChar + return self._current + + @property + def marker(self): # type: () -> int + return self._marker + + def extract(self): # type: () -> unicode + """ + Extracts the value between marker and index + """ + return self[self._marker : self._idx] + + def inc(self, exception=None): # type: (Optional[ParseError.__class__]) -> bool + """ + Increments the parser if the end of the input has not been reached. + Returns whether or not it was able to advance. + """ + try: + self._idx, self._current = next(self._chars) + + return True + except StopIteration: + self._idx = len(self) + self._current = self.EOF + if exception: + raise self.parse_error(exception) + + return False + + def inc_n(self, n, exception=None): # type: (int, Exception) -> bool + """ + Increments the parser by n characters + if the end of the input has not been reached. + """ + for _ in range(n): + if not self.inc(exception=exception): + return False + + return True + + def consume(self, chars, min=0, max=-1): + """ + Consume chars until min/max is satisfied is valid. + """ + while self.current in chars and max != 0: + min -= 1 + max -= 1 + if not self.inc(): + break + + # failed to consume minimum number of characters + if min > 0: + self.parse_error(UnexpectedCharError) + + def end(self): # type: () -> bool + """ + Returns True if the parser has reached the end of the input. + """ + return self._current is self.EOF + + def mark(self): # type: () -> None + """ + Sets the marker to the index's current position + """ + self._marker = self._idx + + def parse_error( + self, exception=ParseError, *args + ): # type: (ParseError.__class__, ...) -> ParseError + """ + Creates a generic "parse error" at the current position. + """ + line, col = self._to_linecol() + + return exception(line, col, *args) + + def _to_linecol(self): # type: () -> Tuple[int, int] + cur = 0 + for i, line in enumerate(self.splitlines()): + if cur + len(line) + 1 > self.idx: + return (i + 1, self.idx - cur) + + cur += len(line) + 1 + + return len(self.splitlines()), 0 diff --git a/pipenv/vendor/tomlkit/toml_char.py b/pipenv/vendor/tomlkit/toml_char.py index 5164ea8b..02c55172 100644 --- a/pipenv/vendor/tomlkit/toml_char.py +++ b/pipenv/vendor/tomlkit/toml_char.py @@ -16,44 +16,51 @@ class TOMLChar(unicode): if len(self) > 1: raise ValueError("A TOML character must be of length 1") + BARE = string.ascii_letters + string.digits + "-_" + KV = "= \t" + NUMBER = string.digits + "+-_.e" + SPACES = " \t" + NL = "\n\r" + WS = SPACES + NL + @lru_cache(maxsize=None) def is_bare_key_char(self): # type: () -> bool """ Whether the character is a valid bare key name or not. """ - return self in string.ascii_letters + string.digits + "-" + "_" + return self in self.BARE @lru_cache(maxsize=None) def is_kv_sep(self): # type: () -> bool """ Whether the character is a valid key/value separator ot not. """ - return self in "= \t" + return self in self.KV @lru_cache(maxsize=None) def is_int_float_char(self): # type: () -> bool """ Whether the character if a valid integer or float value character or not. """ - return self in string.digits + "+" + "-" + "_" + "." + "e" + return self in self.NUMBER @lru_cache(maxsize=None) def is_ws(self): # type: () -> bool """ Whether the character is a whitespace character or not. """ - return self in " \t\r\n" + return self in self.WS @lru_cache(maxsize=None) def is_nl(self): # type: () -> bool """ Whether the character is a new line character or not. """ - return self in "\n\r" + return self in self.NL @lru_cache(maxsize=None) def is_spaces(self): # type: () -> bool """ Whether the character is a space or not """ - return self in " \t" + return self in self.SPACES diff --git a/pipenv/vendor/tomlkit/toml_file.py b/pipenv/vendor/tomlkit/toml_file.py index 631e9959..3b416664 100644 --- a/pipenv/vendor/tomlkit/toml_file.py +++ b/pipenv/vendor/tomlkit/toml_file.py @@ -1,5 +1,8 @@ import io +from typing import Any +from typing import Dict + from .api import loads from .toml_document import TOMLDocument diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 45ff0384..3228068f 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -51,3 +51,4 @@ git+https://github.com/sarugaku/passa.git@master#egg=passa cursor==1.2.0 resolvelib==0.2.2 backports.functools_lru_cache==1.5 +tomlkit \ No newline at end of file From 1f7c9ef949e7d68b6dd642158ee35772614cb2e0 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 10 Nov 2018 12:48:26 +0800 Subject: [PATCH 138/218] update patch --- pipenv/utils.py | 7 +- pipenv/vendor/tomlkit/api.py | 2 - pipenv/vendor/tomlkit/container.py | 16 +- pipenv/vendor/tomlkit/exceptions.py | 2 - pipenv/vendor/tomlkit/items.py | 13 +- pipenv/vendor/tomlkit/parser.py | 7 - pipenv/vendor/tomlkit/source.py | 2 - pipenv/vendor/tomlkit/toml_file.py | 3 - pipenv/vendor/vendor.txt | 3 +- .../vendor/tomlkit-dump-inline-table.patch | 34 ----- .../patches/vendor/tomlkit-fix.patch | 144 ++++++++++++++++++ .../vendor/tomlkit-typing-imports.patch | 93 ----------- .../patches/vendor/tomlkit-update-items.patch | 27 ---- 13 files changed, 158 insertions(+), 195 deletions(-) delete mode 100644 tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch create mode 100644 tasks/vendoring/patches/vendor/tomlkit-fix.patch delete mode 100644 tasks/vendoring/patches/vendor/tomlkit-typing-imports.patch delete mode 100644 tasks/vendoring/patches/vendor/tomlkit-update-items.patch diff --git a/pipenv/utils.py b/pipenv/utils.py index 9d495fa3..9a74b2b9 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -102,19 +102,14 @@ def convert_toml_outline_tables(parsed): else: empty_inline_table = toml.TomlDecoder().get_empty_inline_table for section in ("packages", "dev-packages"): - has_outline_table = False - table_data = parsed.get(section, {}).copy() + table_data = parsed.get(section, {}) for package, value in table_data.items(): if hasattr(value, "keys") and not isinstance( value, (tomlkit.items.InlineTable, toml.decoder.InlineTableDict) ): - has_outline_table = True table = empty_inline_table() table.update(value) table_data[package] = table - if has_outline_table: - # We'll lose comments here, only update when necessary - parsed[section] = table_data return parsed diff --git a/pipenv/vendor/tomlkit/api.py b/pipenv/vendor/tomlkit/api.py index e541c20c..0ac26752 100644 --- a/pipenv/vendor/tomlkit/api.py +++ b/pipenv/vendor/tomlkit/api.py @@ -1,7 +1,5 @@ import datetime as _datetime -from typing import Tuple - from ._utils import parse_rfc3339 from .container import Container from .items import AoT diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index cb8af1d5..9b5db5cb 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -1,13 +1,5 @@ from __future__ import unicode_literals -from typing import Any -from typing import Dict -from typing import Generator -from typing import List -from typing import Optional -from typing import Tuple -from typing import Union - from ._compat import decode from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey @@ -17,6 +9,7 @@ from .items import Item from .items import Key from .items import Null from .items import Table +from .items import Trivia from .items import Whitespace from .items import item as _item @@ -221,7 +214,12 @@ class Container(dict): for i in idx: self._body[i] = (None, Null()) else: - self._body[idx] = (None, Null()) + old_data = self._body[idx][1] + trivia = getattr(old_data, "trivia", None) + if trivia and trivia.comment: + self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) + else: + self._body[idx] = (None, Null()) super(Container, self).__delitem__(key.key) diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py index 4fbc667b..c1a4e620 100644 --- a/pipenv/vendor/tomlkit/exceptions.py +++ b/pipenv/vendor/tomlkit/exceptions.py @@ -1,5 +1,3 @@ -from typing import Optional - class TOMLKitError(Exception): diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 375b5f02..7035b69e 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -7,13 +7,6 @@ from datetime import date from datetime import datetime from datetime import time from enum import Enum -from typing import Any -from typing import Dict -from typing import Generator -from typing import List -from typing import Optional -from typing import Union - from ._compat import PY2 from ._compat import decode @@ -25,6 +18,7 @@ if PY2: from pipenv.vendor.backports.functools_lru_cache import lru_cache else: from functools import lru_cache +from toml.decoder import InlineTableDict def item(value, _parent=None): @@ -40,7 +34,10 @@ def item(value, _parent=None): elif isinstance(value, float): return Float(value, Trivia(), str(value)) elif isinstance(value, dict): - val = Table(Container(), Trivia(), False) + if isinstance(value, InlineTableDict): + val = InlineTable(Container(), Trivia()) + else: + val = Table(Container(), Trivia(), False) for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): val[k] = item(v, _parent=val) diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py index 7b948331..3f507bb4 100644 --- a/pipenv/vendor/tomlkit/parser.py +++ b/pipenv/vendor/tomlkit/parser.py @@ -4,13 +4,6 @@ from __future__ import unicode_literals import re import string -from typing import Any -from typing import Generator -from typing import List -from typing import Optional -from typing import Tuple -from typing import Union - from ._compat import chr from ._compat import decode from ._utils import _escaped diff --git a/pipenv/vendor/tomlkit/source.py b/pipenv/vendor/tomlkit/source.py index 1a96e058..dcfdafd0 100644 --- a/pipenv/vendor/tomlkit/source.py +++ b/pipenv/vendor/tomlkit/source.py @@ -4,8 +4,6 @@ from __future__ import unicode_literals import itertools from copy import copy -from typing import Optional -from typing import Tuple from ._compat import PY2 from ._compat import unicode diff --git a/pipenv/vendor/tomlkit/toml_file.py b/pipenv/vendor/tomlkit/toml_file.py index 3b416664..631e9959 100644 --- a/pipenv/vendor/tomlkit/toml_file.py +++ b/pipenv/vendor/tomlkit/toml_file.py @@ -1,8 +1,5 @@ import io -from typing import Any -from typing import Dict - from .api import loads from .toml_document import TOMLDocument diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 3228068f..9924810e 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -34,7 +34,7 @@ requirementslib==1.2.5 pyparsing==2.2.2 pytoml==0.1.19 plette==0.2.2 - tomlkit==0.4.6 + tomlkit==0.5.2 shellingham==1.2.7 six==1.11.0 semver==2.8.1 @@ -51,4 +51,3 @@ git+https://github.com/sarugaku/passa.git@master#egg=passa cursor==1.2.0 resolvelib==0.2.2 backports.functools_lru_cache==1.5 -tomlkit \ No newline at end of file diff --git a/tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch b/tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch deleted file mode 100644 index 8cd6d5ca..00000000 --- a/tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch +++ /dev/null @@ -1,34 +0,0 @@ -diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py -index 781e2e98..c3c2d59f 100644 ---- a/pipenv/vendor/tomlkit/items.py -+++ b/pipenv/vendor/tomlkit/items.py -@@ -21,6 +21,7 @@ if PY2: - from pipenv.vendor.backports.functools_lru_cache import lru_cache - else: - from functools import lru_cache -+from toml.decoder import InlineTableDict - - - def item(value, _parent=None): -@@ -36,7 +37,10 @@ def item(value, _parent=None): - elif isinstance(value, float): - return Float(value, Trivia(), str(value)) - elif isinstance(value, dict): -- val = Table(Container(), Trivia(), False) -+ if isinstance(value, InlineTableDict): -+ val = InlineTable(Container(), Trivia()) -+ else: -+ val = Table(Container(), Trivia(), False) - for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): - val[k] = item(v, _parent=val) - -@@ -1000,6 +1004,9 @@ class InlineTable(Item, dict): - if key is not None: - super(InlineTable, self).__setitem__(key, value) - -+ if hasattr(value, "trivia") and value.trivia.comment: -+ value.trivia.comment = "" -+ - m = re.match("(?s)^[^ ]*([ ]+).*$", self._trivia.indent) - if not m: - return diff --git a/tasks/vendoring/patches/vendor/tomlkit-fix.patch b/tasks/vendoring/patches/vendor/tomlkit-fix.patch new file mode 100644 index 00000000..5dd9d3fe --- /dev/null +++ b/tasks/vendoring/patches/vendor/tomlkit-fix.patch @@ -0,0 +1,144 @@ +diff --git a/pipenv/vendor/tomlkit/api.py b/pipenv/vendor/tomlkit/api.py +index e541c20c..0ac26752 100644 +--- a/pipenv/vendor/tomlkit/api.py ++++ b/pipenv/vendor/tomlkit/api.py +@@ -1,7 +1,5 @@ + import datetime as _datetime + +-from typing import Tuple +- + from ._utils import parse_rfc3339 + from .container import Container + from .items import AoT +diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py +index cb8af1d5..56ee2d62 100644 +--- a/pipenv/vendor/tomlkit/container.py ++++ b/pipenv/vendor/tomlkit/container.py +@@ -1,13 +1,5 @@ + from __future__ import unicode_literals + +-from typing import Any +-from typing import Dict +-from typing import Generator +-from typing import List +-from typing import Optional +-from typing import Tuple +-from typing import Union +- + from ._compat import decode + from .exceptions import KeyAlreadyPresent + from .exceptions import NonExistentKey +@@ -17,6 +9,7 @@ from .items import Item + from .items import Key + from .items import Null + from .items import Table ++from .items import Trivia + from .items import Whitespace + from .items import item as _item + +@@ -221,7 +214,12 @@ class Container(dict): + for i in idx: + self._body[i] = (None, Null()) + else: +- self._body[idx] = (None, Null()) ++ old_data = self._body[idx][1] ++ trivia = getattr(old_data, "trivia", None) ++ if trivia and trivia.comment: ++ self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) ++ else: ++ self._body[idx] = (None, Null()) + + super(Container, self).__delitem__(key.key) + +diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py +index 4fbc667b..c1a4e620 100644 +--- a/pipenv/vendor/tomlkit/exceptions.py ++++ b/pipenv/vendor/tomlkit/exceptions.py +@@ -1,5 +1,3 @@ +-from typing import Optional +- + + class TOMLKitError(Exception): + +diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py +index 375b5f02..7035b69e 100644 +--- a/pipenv/vendor/tomlkit/items.py ++++ b/pipenv/vendor/tomlkit/items.py +@@ -7,13 +7,6 @@ from datetime import date + from datetime import datetime + from datetime import time + from enum import Enum +-from typing import Any +-from typing import Dict +-from typing import Generator +-from typing import List +-from typing import Optional +-from typing import Union +- + + from ._compat import PY2 + from ._compat import decode +@@ -25,6 +18,7 @@ if PY2: + from pipenv.vendor.backports.functools_lru_cache import lru_cache + else: + from functools import lru_cache ++from toml.decoder import InlineTableDict + + + def item(value, _parent=None): +@@ -40,7 +34,10 @@ def item(value, _parent=None): + elif isinstance(value, float): + return Float(value, Trivia(), str(value)) + elif isinstance(value, dict): +- val = Table(Container(), Trivia(), False) ++ if isinstance(value, InlineTableDict): ++ val = InlineTable(Container(), Trivia()) ++ else: ++ val = Table(Container(), Trivia(), False) + for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): + val[k] = item(v, _parent=val) + +diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py +index 7b948331..3f507bb4 100644 +--- a/pipenv/vendor/tomlkit/parser.py ++++ b/pipenv/vendor/tomlkit/parser.py +@@ -4,13 +4,6 @@ from __future__ import unicode_literals + import re + import string + +-from typing import Any +-from typing import Generator +-from typing import List +-from typing import Optional +-from typing import Tuple +-from typing import Union +- + from ._compat import chr + from ._compat import decode + from ._utils import _escaped +diff --git a/pipenv/vendor/tomlkit/source.py b/pipenv/vendor/tomlkit/source.py +index 1a96e058..dcfdafd0 100644 +--- a/pipenv/vendor/tomlkit/source.py ++++ b/pipenv/vendor/tomlkit/source.py +@@ -4,8 +4,6 @@ from __future__ import unicode_literals + import itertools + + from copy import copy +-from typing import Optional +-from typing import Tuple + + from ._compat import PY2 + from ._compat import unicode +diff --git a/pipenv/vendor/tomlkit/toml_file.py b/pipenv/vendor/tomlkit/toml_file.py +index 3b416664..631e9959 100644 +--- a/pipenv/vendor/tomlkit/toml_file.py ++++ b/pipenv/vendor/tomlkit/toml_file.py +@@ -1,8 +1,5 @@ + import io + +-from typing import Any +-from typing import Dict +- + from .api import loads + from .toml_document import TOMLDocument + diff --git a/tasks/vendoring/patches/vendor/tomlkit-typing-imports.patch b/tasks/vendoring/patches/vendor/tomlkit-typing-imports.patch deleted file mode 100644 index 2288b513..00000000 --- a/tasks/vendoring/patches/vendor/tomlkit-typing-imports.patch +++ /dev/null @@ -1,93 +0,0 @@ -diff --git a/pipenv/vendor/tomlkit/api.py b/pipenv/vendor/tomlkit/api.py -index e541c20c..0ac26752 100644 ---- a/pipenv/vendor/tomlkit/api.py -+++ b/pipenv/vendor/tomlkit/api.py -@@ -1,7 +1,5 @@ - import datetime as _datetime - --from typing import Tuple -- - from ._utils import parse_rfc3339 - from .container import Container - from .items import AoT -diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py -index c1d2d7c6..a7876ff1 100644 ---- a/pipenv/vendor/tomlkit/container.py -+++ b/pipenv/vendor/tomlkit/container.py -@@ -1,13 +1,5 @@ - from __future__ import unicode_literals - --from typing import Any --from typing import Dict --from typing import Generator --from typing import List --from typing import Optional --from typing import Tuple --from typing import Union -- - from ._compat import decode - from .exceptions import KeyAlreadyPresent - from .exceptions import NonExistentKey -diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py -index 8d48bf19..d889a924 100644 ---- a/pipenv/vendor/tomlkit/exceptions.py -+++ b/pipenv/vendor/tomlkit/exceptions.py -@@ -1,6 +1,3 @@ --from typing import Optional -- -- - class TOMLKitError(Exception): - - pass -diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py -index 747dbd50..8807f4b3 100644 ---- a/pipenv/vendor/tomlkit/items.py -+++ b/pipenv/vendor/tomlkit/items.py -@@ -6,14 +6,11 @@ import string - from datetime import date - from datetime import datetime - from datetime import time --from enum import Enum --from typing import Any --from typing import Dict --from typing import Generator --from typing import List --from typing import Optional --from typing import Union -- -+import sys -+if sys.version_info >= (3, 4): -+ from enum import Enum -+else: -+ from pipenv.vendor.backports.enum import Enum - - from ._compat import PY2 - from ._compat import decode -diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py -index b55a3fe4..3d4984d1 100644 ---- a/pipenv/vendor/tomlkit/parser.py -+++ b/pipenv/vendor/tomlkit/parser.py -@@ -7,10 +7,6 @@ import re - import string - - from copy import copy --from typing import Iterator --from typing import Optional --from typing import Tuple --from typing import Union - - from ._compat import PY2 - from ._compat import chr -diff --git a/pipenv/vendor/tomlkit/toml_file.py b/pipenv/vendor/tomlkit/toml_file.py -index 3b416664..631e9959 100644 ---- a/pipenv/vendor/tomlkit/toml_file.py -+++ b/pipenv/vendor/tomlkit/toml_file.py -@@ -1,8 +1,5 @@ - import io - --from typing import Any --from typing import Dict -- - from .api import loads - from .toml_document import TOMLDocument - diff --git a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch b/tasks/vendoring/patches/vendor/tomlkit-update-items.patch deleted file mode 100644 index ed2fb95e..00000000 --- a/tasks/vendoring/patches/vendor/tomlkit-update-items.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py -index 37014921..987a0790 100644 ---- a/pipenv/vendor/tomlkit/container.py -+++ b/pipenv/vendor/tomlkit/container.py -@@ -9,6 +9,7 @@ from .items import Item - from .items import Key - from .items import Null - from .items import Table -+from .items import Trivia - from .items import Whitespace - from .items import item as _item - -@@ -189,8 +190,12 @@ class Container(dict): - if idx is None: - raise NonExistentKey(key) - -- self._body[idx] = (None, Null()) -- -+ old_data = self._body[idx][1] -+ trivia = getattr(old_data, "trivia", None) -+ if trivia and getattr(trivia, "comment", None): -+ self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) -+ else: -+ self._body[idx] = (None, Null()) - super(Container, self).__delitem__(key.key) - - return self From de98b874ec22acdedc1155fa9e6db3ac40b2a016 Mon Sep 17 00:00:00 2001 From: frostming Date: Sat, 10 Nov 2018 14:51:19 +0800 Subject: [PATCH 139/218] backports import --- pipenv/vendor/tomlkit/items.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 7035b69e..39128d30 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -6,7 +6,10 @@ import string from datetime import date from datetime import datetime from datetime import time -from enum import Enum +try: + from enum import Enum +except ImportError: + from pipenv.vendor.backports.enum import Enum from ._compat import PY2 from ._compat import decode From f5c7c58be01e4a0fa17c64d5790e16c8cfc56bec Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 10 Nov 2018 15:41:09 +0800 Subject: [PATCH 140/218] Change fallback style --- pipenv/vendor/tomlkit/items.py | 6 +-- .../patches/vendor/tomlkit-fix.patch | 50 +++++++++++-------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 39128d30..cccfd4a1 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -6,10 +6,6 @@ import string from datetime import date from datetime import datetime from datetime import time -try: - from enum import Enum -except ImportError: - from pipenv.vendor.backports.enum import Enum from ._compat import PY2 from ._compat import decode @@ -18,8 +14,10 @@ from ._compat import unicode from ._utils import escape_string if PY2: + from pipenv.vendor.backports.enum import Enum from pipenv.vendor.backports.functools_lru_cache import lru_cache else: + from enum import Enum from functools import lru_cache from toml.decoder import InlineTableDict diff --git a/tasks/vendoring/patches/vendor/tomlkit-fix.patch b/tasks/vendoring/patches/vendor/tomlkit-fix.patch index 5dd9d3fe..36e2f808 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-fix.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-fix.patch @@ -4,19 +4,19 @@ index e541c20c..0ac26752 100644 +++ b/pipenv/vendor/tomlkit/api.py @@ -1,7 +1,5 @@ import datetime as _datetime - + -from typing import Tuple - from ._utils import parse_rfc3339 from .container import Container from .items import AoT diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py -index cb8af1d5..56ee2d62 100644 +index cb8af1d5..9b5db5cb 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -1,13 +1,5 @@ from __future__ import unicode_literals - + -from typing import Any -from typing import Dict -from typing import Generator @@ -35,7 +35,7 @@ index cb8af1d5..56ee2d62 100644 +from .items import Trivia from .items import Whitespace from .items import item as _item - + @@ -221,7 +214,12 @@ class Container(dict): for i in idx: self._body[i] = (None, Null()) @@ -47,9 +47,9 @@ index cb8af1d5..56ee2d62 100644 + self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) + else: + self._body[idx] = (None, Null()) - + super(Container, self).__delitem__(key.key) - + diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py index 4fbc667b..c1a4e620 100644 --- a/pipenv/vendor/tomlkit/exceptions.py @@ -57,17 +57,18 @@ index 4fbc667b..c1a4e620 100644 @@ -1,5 +1,3 @@ -from typing import Optional - - + class TOMLKitError(Exception): - + diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py -index 375b5f02..7035b69e 100644 +index 375b5f02..cccfd4a1 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py -@@ -7,13 +7,6 @@ from datetime import date +@@ -6,14 +6,6 @@ import string + from datetime import date from datetime import datetime from datetime import time - from enum import Enum +-from enum import Enum -from typing import Any -from typing import Dict -from typing import Generator @@ -75,18 +76,23 @@ index 375b5f02..7035b69e 100644 -from typing import Optional -from typing import Union - - + from ._compat import PY2 from ._compat import decode -@@ -25,6 +18,7 @@ if PY2: +@@ -22,9 +14,12 @@ from ._compat import unicode + from ._utils import escape_string + + if PY2: ++ from pipenv.vendor.backports.enum import Enum from pipenv.vendor.backports.functools_lru_cache import lru_cache else: ++ from enum import Enum from functools import lru_cache +from toml.decoder import InlineTableDict - - + + def item(value, _parent=None): -@@ -40,7 +34,10 @@ def item(value, _parent=None): +@@ -40,7 +35,10 @@ def item(value, _parent=None): elif isinstance(value, float): return Float(value, Trivia(), str(value)) elif isinstance(value, dict): @@ -97,7 +103,7 @@ index 375b5f02..7035b69e 100644 + val = Table(Container(), Trivia(), False) for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): val[k] = item(v, _parent=val) - + diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py index 7b948331..3f507bb4 100644 --- a/pipenv/vendor/tomlkit/parser.py @@ -105,7 +111,7 @@ index 7b948331..3f507bb4 100644 @@ -4,13 +4,6 @@ from __future__ import unicode_literals import re import string - + -from typing import Any -from typing import Generator -from typing import List @@ -122,11 +128,11 @@ index 1a96e058..dcfdafd0 100644 +++ b/pipenv/vendor/tomlkit/source.py @@ -4,8 +4,6 @@ from __future__ import unicode_literals import itertools - + from copy import copy -from typing import Optional -from typing import Tuple - + from ._compat import PY2 from ._compat import unicode diff --git a/pipenv/vendor/tomlkit/toml_file.py b/pipenv/vendor/tomlkit/toml_file.py @@ -135,10 +141,10 @@ index 3b416664..631e9959 100644 +++ b/pipenv/vendor/tomlkit/toml_file.py @@ -1,8 +1,5 @@ import io - + -from typing import Any -from typing import Dict - from .api import loads from .toml_document import TOMLDocument - + From 2b90c89d1f518a543f775cb811ee8b5565ee1366 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 16:30:42 -0500 Subject: [PATCH 141/218] Revendor requirementslib - Implement improvements and bugfixes in codebase - Remote archives will now resolve properly Signed-off-by: Dan Ryan --- pipenv/__init__.py | 6 +- pipenv/_compat.py | 2 +- pipenv/core.py | 15 +- pipenv/project.py | 105 ++--- pipenv/utils.py | 14 +- pipenv/vendor/requirementslib/__init__.py | 4 + pipenv/vendor/requirementslib/exceptions.py | 75 ++++ .../vendor/requirementslib/models/lockfile.py | 68 +++- .../vendor/requirementslib/models/pipfile.py | 44 +- .../requirementslib/models/requirements.py | 340 +++++++++++----- .../requirementslib/models/setup_info.py | 378 ++++++++++++++++++ pipenv/vendor/requirementslib/models/utils.py | 41 +- pipenv/vendor/requirementslib/models/vcs.py | 3 +- tests/integration/conftest.py | 13 + tests/integration/test_uninstall.py | 6 +- 15 files changed, 944 insertions(+), 170 deletions(-) create mode 100644 pipenv/vendor/requirementslib/models/setup_info.py diff --git a/pipenv/__init__.py b/pipenv/__init__.py index f8a1a8b3..ba4dd9c3 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -10,7 +10,7 @@ import warnings from .__version__ import __version__ -PIPENV_ROOT = os.path.dirname(os.path.realpath(__file__)) +PIPENV_ROOT = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) PIPENV_VENDOR = os.sep.join([PIPENV_ROOT, "vendor"]) PIPENV_PATCHED = os.sep.join([PIPENV_ROOT, "patched"]) # Inject vendored directory into system path. @@ -27,11 +27,13 @@ warnings.filterwarnings("ignore", category=UserWarning) if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): if sys.stdout.isatty() and sys.stderr.isatty(): import io + import atexit sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') + atexit.register(sys.stdout.close) sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') + atexit.register(sys.stdout.close) os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") -os.environ["PIP_SHIMS_BASE_MODULE"] = fs_str("pipenv.patched.notpip") # Hack to make things work better. try: diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 6e5ae6a7..fb2c0147 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -382,7 +382,7 @@ def decode_output(output): except (AttributeError, UnicodeDecodeError, UnicodeEncodeError): if six.PY2: output = unicode.translate(vistir.misc.to_text(output), - UNICODE_TO_ASCII_TRANSLATION_MAP) + UNICODE_TO_ASCII_TRANSLATION_MAP) else: output = output.translate(UNICODE_TO_ASCII_TRANSLATION_MAP) output = output.encode(DEFAULT_ENCODING, "replace") diff --git a/pipenv/core.py b/pipenv/core.py index f9b8eaee..3cbd1645 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -40,8 +40,8 @@ from .utils import ( clean_resolved_dep, parse_indexes, escape_cmd, - fix_venv_site, create_spinner, + get_canonical_names ) from . import environments, pep508checker, progress from .environments import ( @@ -1296,7 +1296,7 @@ def pip_install( pypi_mirror=None, trusted_hosts=None ): - from notpip._internal import logger as piplogger + from pipenv.patched.notpip._internal import logger as piplogger from .utils import Mapping from .vendor.urllib3.util import parse_url @@ -1746,11 +1746,11 @@ def do_install( if requirements or package_args or project.pipfile_exists: skip_requirements = True # Don't attempt to install develop and default packages if Pipfile is missing - if not project.pipfile_exists and not (packages or dev) and not code: - if not (skip_lock or deploy): - raise exceptions.PipfileNotFound(project.pipfile_location) - elif (skip_lock or deploy) and not project.lockfile_exists: - raise exceptions.LockfileNotFound(project.lockfile_location) + if not project.pipfile_exists and not (package_args or dev) and not code: + if not (ignore_pipfile or deploy): + raise exceptions.PipfileNotFound(project.path_to("Pipfile")) + elif ((skip_lock and deploy) or ignore_pipfile) and not project.lockfile_exists: + raise exceptions.LockfileNotFound(project.path_to("Pipfile.lock")) concurrent = not sequential # Ensure that virtualenv is available. ensure_project( @@ -2092,7 +2092,6 @@ def do_uninstall( ) ) package_names = develop - fix_venv_site(project.env_paths["lib"]) # Remove known "bad packages" from the list. bad_pkgs = set([canonicalize_name(pkg) for pkg in BAD_PACKAGES]) for bad_package in BAD_PACKAGES: diff --git a/pipenv/project.py b/pipenv/project.py index 26b4cf0c..d4713b89 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -804,7 +804,7 @@ class Project(object): .lstrip("\n") .split("\n") ) - sources = [DEFAULT_SOURCE] + sources = [DEFAULT_SOURCE,] for i, index in enumerate(indexes): if not index: continue @@ -831,55 +831,68 @@ class Project(object): version = python_version(required_python) or PIPENV_DEFAULT_PYTHON_VERSION if version and len(version) >= 3: data[u"requires"] = {"python_version": version[: len("2.7")]} - self.write_toml(data, "Pipfile") + self.write_toml(data) def get_or_create_lockfile(self): - from requirementslib.models.lockfile import Lockfile as Req_Lockfile + from pipenv.vendor.requirementslib.models.lockfile import Lockfile as Req_Lockfile lockfile = None - try: - lockfile = Req_Lockfile.load(self.lockfile_location) - except OSError: - lockfile = Req_Lockfile(self.lockfile_content) - return lockfile + if self.lockfile_exists: + try: + lockfile = Req_Lockfile.load(self.lockfile_location) + except OSError: + lockfile = Req_Lockfile.from_data(self.lockfile_location, self.lockfile_content) else: - if lockfile._lockfile is not None: - return lockfile - if self.lockfile_exists and self.lockfile_content: - from .vendor.plette.lockfiles import Lockfile - lockfile_dict = self.lockfile_content.copy() - sources = lockfile_dict["_meta"].get("sources", []) - if not sources: - sources = self.pipfile_sources - elif not isinstance(sources, list): - sources = [sources,] - lockfile_dict["_meta"]["sources"] = [ - { - "name": s["name"], - "url": s["url"], - "verify_ssl": ( - s["verify_ssl"] if isinstance(s["verify_ssl"], bool) else ( - True if s["verify_ssl"].lower() == "true" else False - ) + lockfile = Req_Lockfile.from_data(path=self.lockfile_location, data=self._lockfile, meta_from_project=False) + if lockfile._lockfile is not None: + return lockfile + if self.lockfile_exists and self.lockfile_content: + lockfile_dict = self.lockfile_content.copy() + sources = lockfile_dict.get("_meta", {}).get("sources", []) + if not sources: + sources = self.pipfile_sources + elif not isinstance(sources, list): + sources = [sources,] + lockfile_dict["_meta"]["sources"] = [ + { + "name": s["name"], + "url": s["url"], + "verify_ssl": ( + s["verify_ssl"] if isinstance(s["verify_ssl"], bool) else ( + True if s["verify_ssl"].lower() == "true" else False ) - } for s in sources - ] - _created_lockfile = Lockfile(lockfile_dict) - lockfile._lockfile = lockfile.projectfile.model = _created_lockfile - return lockfile - elif self.pipfile_exists: - from .vendor.plette.lockfiles import Lockfile, PIPFILE_SPEC_CURRENT - lockfile_dict = { - "_meta": { - "hash": {"sha256": self.calculate_pipfile_hash()}, - "pipfile-spec": PIPFILE_SPEC_CURRENT, - "sources": self.pipfile_sources, - "requires": self.parsed_pipfile.get("requires", {}) - }, - "default": self._lockfile["default"].copy(), - "develop": self._lockfile["develop"].copy() - } - lockfile._lockfile = Lockfile(lockfile_dict) - return lockfile + ) + } for s in sources + ] + _created_lockfile = Req_Lockfile.from_data( + path=self.lockfile_location, data=lockfile_dict, meta_from_project=False + ) + lockfile._lockfile = lockfile.projectfile.model = _created_lockfile + return lockfile + elif self.pipfile_exists: + lockfile_dict = { + "default": self._lockfile["default"].copy(), + "develop": self._lockfile["develop"].copy() + } + lockfile_dict.update({"_meta": self.get_lockfile_meta()}) + _created_lockfile = Req_Lockfile.from_data( + path=self.lockfile_location, data=lockfile_dict, meta_from_project=False + ) + lockfile._lockfile = _created_lockfile + return lockfile + + def get_lockfile_meta(self): + from .vendor.plette.lockfiles import PIPFILE_SPEC_CURRENT + sources = self.lockfile_content.get("_meta", {}).get("sources", []) + if not sources: + sources = self.pipfile_sources + elif not isinstance(sources, list): + sources = [sources,] + return { + "hash": {"sha256": self.calculate_pipfile_hash()}, + "pipfile-spec": PIPFILE_SPEC_CURRENT, + "sources": sources, + "requires": self.parsed_pipfile.get("requires", {}) + } def write_toml(self, data, path=None): """Writes the given data structure out as TOML.""" @@ -943,7 +956,7 @@ class Project(object): @property def sources(self): if self.lockfile_exists and hasattr(self.lockfile_content, "keys"): - meta_ = self.lockfile_content["_meta"] + meta_ = self.lockfile_content.get("_meta", {}) sources_ = meta_.get("sources") if sources_: return sources_ diff --git a/pipenv/utils.py b/pipenv/utils.py index c9feeafd..3a9ef307 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -18,7 +18,8 @@ from first import first from vistir.misc import fs_str six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) -from six.moves import Mapping +six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) +from six.moves import Mapping, Sequence from vistir.compat import ResourceWarning @@ -1035,6 +1036,17 @@ def path_to_url(path): return Path(normalize_drive(os.path.abspath(path))).as_uri() +def get_canonical_names(packages): + """Canonicalize a list of packages and return a set of canonical names""" + from .vendor.packaging.utils import canonicalize_name + + if not isinstance(packages, Sequence): + if not isinstance(packages, six.string_types): + return packages + packages = [packages,] + return set([canonicalize_name(pkg) for pkg in packages if pkg]) + + def walk_up(bottom): """Mimic os.walk, but walk 'up' instead of down the directory tree. From: https://gist.github.com/zdavkeos/1098474 diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index edbab5bc..881e9ac9 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -2,6 +2,10 @@ __version__ = '1.2.6' import logging +import warnings + +warnings.filterwarnings("ignore", category=ResourceWarning) + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/vendor/requirementslib/exceptions.py index de8bf8ef..1a73f98e 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/vendor/requirementslib/exceptions.py @@ -1,7 +1,12 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import import errno +import os import six +import sys + + +from vistir.compat import FileNotFoundError if six.PY2: @@ -15,3 +20,73 @@ else: class RequirementError(Exception): pass + + +class MissingParameter(Exception): + def __init__(self, param): + super(Exception, self).__init__() + print("Missing parameter: %s" % param, file=sys.stderr, flush=True) + + +class FileCorruptException(OSError): + def __init__(self, path, *args, **kwargs): + path = path + backup_path = kwargs.pop("backup_path", None) + if not backup_path and args: + args = reversed(args) + backup_path = args.pop() + if not isinstance(backup_path, six.string_types) or not os.path.exists(os.path.abspath(os.path.dirname(backup_path))): + args.append(backup_path) + backup_path = None + if args: + args = reversed(args) + self.path = path + self.backup_path = backup_path + self.show(self.path, self.backup_path) + super(OSError, self).__init__(path, *args, **kwargs) + + @classmethod + def show(cls, path, backup_path=None): + print("ERROR: Failed to load file at %s" % path, file=sys.stderr, flush=True) + if backup_path: + msg = "it will be backed up to %s and removed" % backup_path + else: + msg = "it will be removed and replaced." + print("The file is corrupt, %s" % msg, file=sys.stderr, flush=True) + + +class LockfileCorruptException(FileCorruptException): + + @classmethod + def show(cls, path, backup_path=None): + print("ERROR: Failed to load lockfile at %s" % path, file=sys.stderr, flush=True) + if backup_path: + msg = "it will be backed up to %s and removed" % backup_path + else: + msg = "it will be removed and replaced on the next lock." + print("Your lockfile is corrupt, %s" % msg, file=sys.stderr, flush=True) + + +class PipfileCorruptException(FileCorruptException): + + @classmethod + def show(cls, path, backup_path=None): + print("ERROR: Failed to load Pipfile at %s" % path, file=sys.stderr, flush=True) + if backup_path: + msg = "it will be backed up to %s and removed" % backup_path + else: + msg = "it will be removed and replaced on the next lock." + print("Your Pipfile is corrupt, %s" % msg, file=sys.stderr, flush=True) + + +class PipfileNotFound(FileNotFoundError): + def __init__(self, path, *args, **kwargs): + self.errno = errno.ENOENT + self.path = path + self.show(path) + super(PipfileNotFound, self).__init__(*args, **kwargs) + + @classmethod + def show(cls, path): + print("ERROR: The file could not be found: %s" % path, file=sys.stderr, flush=True) + print("Aborting...", file=sys.stderr, flush=True) diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index 3e482813..6f61f57e 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -9,12 +9,13 @@ import itertools import plette.lockfiles import six -from vistir.compat import Path, FileNotFoundError +from vistir.compat import Path, FileNotFoundError, JSONDecodeError from .project import ProjectFile from .requirements import Requirement from .utils import optional_instance_of +from ..exceptions import LockfileCorruptException, PipfileNotFound, MissingParameter from ..utils import is_vcs, is_editable, merge_items DEFAULT_NEWLINES = u"\n" @@ -134,7 +135,17 @@ class Lockfile(object): return pf @classmethod - def load_projectfile(cls, path, create=True): + def lockfile_from_pipfile(cls, pipfile_path): + from .pipfile import Pipfile + if os.path.isfile(pipfile_path): + if not os.path.isabs(pipfile_path): + pipfile_path = os.path.abspath(pipfile_path) + pipfile = Pipfile.load(os.path.dirname(pipfile_path)) + return plette.lockfiles.Lockfile.with_meta_from(pipfile._pipfile) + raise PipfileNotFound(pipfile_path) + + @classmethod + def load_projectfile(cls, path, create=True, data=None): """Given a path, load or create the necessary lockfile. :param str path: Path to the project root or lockfile @@ -155,8 +166,48 @@ class Lockfile(object): elif not lockfile_path.exists() and not create: raise FileNotFoundError("Lockfile does not exist: %s" % lockfile_path.as_posix()) projectfile = cls.read_projectfile(lockfile_path.as_posix()) + if not lockfile_path.exists(): + if not data: + lf = cls.lockfile_from_pipfile(project_path.joinpath("Pipfile")) + else: + lf = plette.lockfiles.Lockfile(data) + projectfile.model = lf return projectfile + @classmethod + def from_data(cls, path, data, meta_from_project=True): + """Create a new lockfile instance from a dictionary. + + :param str path: Path to the project root. + :param dict data: Data to load into the lockfile. + :param bool meta_from_project: Attempt to populate the meta section from the + project root, default True. + """ + + if path is None: + raise MissingParameter("path") + if data is None: + raise MissingParameter("data") + if not isinstance(data, dict): + raise TypeError("Expecting a dictionary for parameter 'data'") + path = os.path.abspath(str(path)) + if os.path.isdir(path): + project_path = path + elif not os.path.isdir(path) and os.path.isdir(os.path.dirname(path)): + project_path = os.path.dirname(path) + pipfile_path = os.path.join(project_path, "Pipfile") + lockfile_path = os.path.join(project_path, "Pipfile.lock") + if meta_from_project: + lockfile = cls.lockfile_from_pipfile(pipfile_path) + lockfile.update(data) + else: + lockfile = plette.lockfiles.Lockfile(data) + projectfile = ProjectFile(line_ending=DEFAULT_NEWLINES, location=lockfile_path, model=lockfile) + return cls( + projectfile=projectfile, lockfile=lockfile, + newlines=projectfile.line_ending, path=Path(projectfile.location) + ) + @classmethod def load(cls, path, create=True): """Create a new lockfile instance. @@ -170,7 +221,18 @@ class Lockfile(object): :rtype: :class:`~requirementslib.models.lockfile.Lockfile` """ - projectfile = cls.load_projectfile(path, create=create) + try: + projectfile = cls.load_projectfile(path, create=create) + except JSONDecodeError as e: + path = os.path.abspath(path) + if not os.path.isdir(path): + path = os.path.dirname(path) + path = Path(os.path.join(path, "Pipfile.lock")) + formatted_path = path.as_posix() + backup_path = "%.bak" % formatted_path + LockfileCorruptException.show(formatted_path, backup_path=backup_path) + path.rename(backup_path) + cls.load(formatted_path, create=True) lockfile_path = Path(projectfile.location) creation_args = { "projectfile": projectfile, diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 58d54055..dbb024be 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -23,13 +23,26 @@ is_path = optional_instance_of(Path) is_projectfile = optional_instance_of(ProjectFile) +def reorder_source_keys(data): + for i, entry in enumerate(data["source"]): + table = tomlkit.table() + table["name"] = entry["name"] + table["url"] = entry["url"] + table["verify_ssl"] = entry["verify_ssl"] + data["source"][i] = table + return data + + class PipfileLoader(plette.pipfiles.Pipfile): @classmethod def validate(cls, data): for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items(): if key not in data or key == "source": continue - klass.validate(data[key]) + try: + klass.validate(data[key]) + except Exception: + pass @classmethod def load(cls, f, encoding=None): @@ -37,19 +50,26 @@ class PipfileLoader(plette.pipfiles.Pipfile): if encoding is not None: content = content.decode(encoding) _data = tomlkit.loads(content) + _data["source"] = _data.get("source", []) + _data.get("sources", []) + _data = reorder_source_keys(_data) if "source" not in _data: - if "sources" in _data: - _data["source"] = _data["sources"] - content = tomlkit.dumps(_data) - else: - # HACK: There is no good way to prepend a section to an existing - # TOML document, but there's no good way to copy non-structural - # content from one TOML document to another either. Modify the - # TOML content directly, and load the new in-memory document. - sep = "" if content.startswith("\n") else "\n" - content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content + # HACK: There is no good way to prepend a section to an existing + # TOML document, but there's no good way to copy non-structural + # content from one TOML document to another either. Modify the + # TOML content directly, and load the new in-memory document. + sep = "" if content.startswith("\n") else "\n" + content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content data = tomlkit.loads(content) - return cls(data) + data = reorder_source_keys(data) + instance = cls(data) + new_data = reorder_source_keys(instance._data) + instance._data = new_data + return instance + + def __getattribute__(self, key): + if key == "source": + return self._data[key] + return super(PipfileLoader, self).__getattribute__(key) @attr.s(slots=True) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index a36e5ba4..51411429 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -36,6 +36,7 @@ from ..utils import ( add_ssh_scheme_to_git_uri, strip_ssh_from_git_uri, ) +from .setup_info import SetupInfo from .utils import ( HASH_STRING, build_vcs_link, @@ -46,7 +47,6 @@ from .utils import ( init_requirement, is_pinned_requirement, make_install_requirement, - optional_instance_of, parse_extras, specs_to_string, split_markers_from_line, @@ -56,6 +56,7 @@ from .utils import ( validate_vcs, normalize_name, create_link, + get_pyproject ) @@ -148,13 +149,19 @@ class FileRequirement(object): uri = attr.ib() #: Link object representing the package to clone link = attr.ib() + #: PyProject Requirements + pyproject_requires = attr.ib(default=attr.Factory(list)) + #: PyProject Build System + pyproject_backend = attr.ib(default=None) + #: PyProject Path + pyproject_path = attr.ib(default=None) _has_hashed_name = attr.ib(default=False) #: Package name name = attr.ib() #: A :class:`~pkg_resources.Requirement` isntance req = attr.ib() - #: Whether this is a direct url requirement - is_direct = attr.ib(default=False) + #: Setup metadata e.g. dependencies + setup_info = attr.ib(default=None) @classmethod def get_link_from_line(cls, line): @@ -258,107 +265,110 @@ class FileRequirement(object): return LinkInfo(vcs_type, prefer, relpath, path, uri, link) - def __attrs_post_init__(self): - if self.req and getattr(self.req, "url"): - self.uri = self.req.url + @property + def setup_py_dir(self): + if self.setup_path: + return os.path.dirname(os.path.abspath(self.setup_path)) + + @property + def dependencies(self): + build_deps = [] + setup_deps = [] + deps = {} + if self.setup_info: + setup_info = self.setup_info.as_dict() + deps.update(setup_info.get("requires", {})) + setup_deps.extend(setup_info.get("setup_requires", [])) + build_deps.extend(setup_info.get("build_requires", [])) + if self.pyproject_requires: + build_deps.extend(self.pyproject_requires) + return deps, setup_deps, build_deps @uri.default def get_uri(self): if self.path and not self.uri: self._uri_scheme = "path" - self.uri = pip_shims.shims.path_to_url(os.path.abspath(self.path)) + return pip_shims.shims.path_to_url(os.path.abspath(self.path)) elif self.req and getattr(self.req, "url"): - self.uri = self.req.url + return self.req.url @name.default def get_name(self): loc = self.path or self.uri - if loc: - self._uri_scheme = "path" if self.path else "uri" + if loc and not self._uri_scheme: + self._uri_scheme = "path" if self.path else "file" name = None - if getattr(self, "req", None) and getattr(self.req, "name"): - return self.req.name - if self.link and self.link.egg_fragment: + if getattr(self, "req", None) and getattr(self.req, "name") and self.req.name is not None: + if self.is_direct_url: + return self.req.name + if self.link and self.link.egg_fragment and not self._has_hashed_name: return self.link.egg_fragment elif self.link and self.link.is_wheel: from pip_shims import Wheel - + self._has_hashed_name = False return Wheel(self.link.filename).name - if ( - self._uri_scheme != "uri" - and self.path - and self.setup_path - and self.setup_path.exists() - ): - from setuptools.dist import distutils - - old_curdir = os.path.abspath(os.getcwd()) - try: - os.chdir(str(self.setup_path.parent)) - dist = distutils.core.run_setup(self.setup_path.as_posix()) - name = dist.get_name() - except (FileNotFoundError, IOError) as e: - dist = None - except Exception as e: - from pip_shims.shims import make_abstract_dist - - try: - if not isinstance(Path, self.path): - _path = Path(self.path) - else: - _path = self.path - if self.editable: - _ireq = pip_shims.shims.install_req_from_editable(_path.as_uri()) - else: - _ireq = pip_shims.shims.install_req_from_line(_path.as_posix()) - dist = make_abstract_dist(_ireq).get_dist() - name = dist.project_name - except (TypeError, ValueError, AttributeError) as e: - dist = None - finally: - os.chdir(old_curdir) + elif self.link and ((self.link.scheme == "file" or self.editable) or ( + self.path and self.setup_path and os.path.isfile(str(self.setup_path)) + )): + if self.editable: + line = pip_shims.shims.path_to_url(self.setup_py_dir) + _ireq = pip_shims.shims.install_req_from_editable(line) + else: + _ireq = pip_shims.shims.install_req_from_line(Path(self.setup_py_dir).as_posix()) + from .setup_info import SetupInfo + subdir = getattr(self, "subdirectory", None) + setupinfo = SetupInfo.from_ireq(_ireq, subdir=subdir) + if setupinfo: + self.setup_info = setupinfo + setupinfo_dict = setupinfo.as_dict() + setup_name = setupinfo_dict.get("name", None) + if setup_name: + name = setup_name + self._has_hashed_name = False + version = setupinfo_dict.get("version") + if version and not self.version: + self.version = version + build_requires = setupinfo_dict.get("build_requires") + build_backend = setupinfo_dict.get("build_backend") + if build_requires and not self.pyproject_requires: + self.pyproject_requires = build_requires + if build_backend and not self.pyproject_backend: + self.pyproject_backend = build_backend hashed_loc = hashlib.sha256(loc.encode("utf-8")).hexdigest() hashed_name = hashed_loc[-7:] - if not name or name == "UNKNOWN": + if not name or name.lower() == "unknown": self._has_hashed_name = True name = hashed_name - if self.link and not self._has_hashed_name: + else: + self._has_hashed_name = False + name_in_link = getattr(self.link, "egg_fragment", "") if self.link else "" + if not self._has_hashed_name and name_in_link != name: self.link = create_link("{0}#egg={1}".format(self.link.url, name)) return name @link.default def get_link(self): target = "{0}".format(self.uri) - if hasattr(self, "name"): + if hasattr(self, "name") and not self._has_hashed_name: target = "{0}#egg={1}".format(target, self.name) link = create_link(target) return link @req.default def get_requirement(self): - if self.link.is_artifact and not self.editable: - if self._uri_scheme == "uri": - if self.name: - req_str = "{0} @ {1}".format(self.name, self.link.url_without_fragment) - else: - req_str = "{0}".format(self.link.url_without_fragment) - req = init_requirement(req_str) - req.line = req_str - else: - req = init_requirement(normalize_name(self.name)) - else: - req = init_requirement(normalize_name(self.name)) - req.editable = False - req.line = self.link.url_without_fragment - if self.path and self.link and self.link.scheme.startswith("file"): - req.local_file = True - req.path = self.path + req = init_requirement(normalize_name(self.name)) + req.editable = False + req.line = self.link.url_without_fragment + if self.path and self.link and self.link.scheme.startswith("file"): + req.local_file = True + req.path = self.path + if self.editable: req.url = None - self._uri_scheme = "file" else: - req.local_file = False - req.path = None - if not getattr(req, "url", None): + req.url = self.link.url_without_fragment + else: + req.local_file = False + req.path = None req.url = self.link.url_without_fragment if self.editable: req.editable = True @@ -389,6 +399,99 @@ class FileRequirement(object): return path.as_posix() return + @classmethod + def create( + cls, path=None, uri=None, editable=False, extras=None, link=None, vcs_type=None, + name=None, req=None, line=None, uri_scheme=None, setup_path=None, relpath=None + ): + import pip_shims.shims + if relpath and not path: + path = relpath + if not path and uri and link.scheme == "file": + path = os.path.abspath(pip_shims.shims.url_to_path(unquote(uri))) + try: + path = get_converted_relative_path(path) + except ValueError: # Vistir raises a ValueError if it can't make a relpath + path = path + if line and not (uri_scheme and uri and link): + vcs_type, uri_scheme, relpath, path, uri, link = cls.get_link_from_line(line) + if not uri_scheme: + uri_scheme = "path" if path else "file" + if path and not uri: + uri = unquote(pip_shims.shims.path_to_url(os.path.abspath(path))) + if not link: + link = create_link(uri) + if not uri: + uri = unquote(link.url_without_fragment) + if not extras: + extras = [] + pyproject_path = None + if path is not None: + pyproject_requires = get_pyproject(os.path.abspath(path)) + pyproject_backend = None + pyproject_requires = None + if pyproject_requires is not None: + pyproject_requires, pyproject_backend = pyproject_requires + if path: + pyproject_path = Path(path).joinpath("pyproject.toml") + if not pyproject_path.exists(): + pyproject_path = None + if not setup_path and path is not None: + setup_path = Path(path).joinpath("setup.py") + if setup_path and isinstance(setup_path, Path): + setup_path = setup_path.as_posix() + creation_kwargs = { + "editable": editable, + "extras": extras, + "pyproject_path": pyproject_path, + "setup_path": setup_path if setup_path else None, + "uri_scheme": uri_scheme, + "link": link, + "uri": uri, + "pyproject_requires": pyproject_requires, + "pyproject_backend": pyproject_backend + } + if vcs_type: + creation_kwargs["vcs_type"] = vcs_type + _line = None + if not name: + import pip_shims.shims + _line = unquote(link.url_without_fragment) if link.url else uri + if editable: + ireq = pip_shims.shims.install_req_from_editable(_line) + else: + _line = path if (uri_scheme and uri_scheme == "path") else _line + ireq = pip_shims.shims.install_req_from_line(_line) + setup_info = SetupInfo.from_ireq(ireq) + setupinfo_dict = setup_info.as_dict() + setup_name = setupinfo_dict.get("name", None) + if setup_name: + name = setup_name + build_requires = setupinfo_dict.get("build_requires", []) + build_backend = setupinfo_dict.get("build_backend", []) + if not creation_kwargs.get("pyproject_requires") and build_requires: + creation_kwargs["pyproject_requires"] = build_requires + if not creation_kwargs.get("pyproject_backend") and build_backend: + creation_kwargs["pyproject_backend"] = build_backend + creation_kwargs["setup_info"] = setup_info + if path or relpath: + creation_kwargs["path"] = relpath if relpath else path + if req: + creation_kwargs["req"] = req + if creation_kwargs.get("req") and line and not getattr(creation_kwargs["req"], "line", None): + creation_kwargs["req"].line = line + if name: + creation_kwargs["name"] = name + cls_inst = cls(**creation_kwargs) + if not _line: + if editable and uri_scheme == "path": + _line = relpath if relpath else path + else: + _line = unquote(cls_inst.link.url_without_fragment) or cls_inst.uri + _line = "{0}#egg={1}".format(line, cls_inst.name) if not cls_inst._has_hashed_name else _line + cls_inst.req.line = line if line else _line + return cls_inst + @classmethod def from_line(cls, line): line = line.strip('"').strip("'") @@ -410,7 +513,6 @@ class FileRequirement(object): name = getattr(req, "name", None) line = getattr(req, "url", None) vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line(line) - setup_path = Path(path) / "setup.py" if path else None arg_dict = { "path": relpath if relpath else path, "uri": unquote(link.url_without_fragment), @@ -418,6 +520,7 @@ class FileRequirement(object): "editable": editable, "setup_path": setup_path, "uri_scheme": prefer, + "line": line } if link and link.is_wheel: from pip_shims import Wheel @@ -427,10 +530,7 @@ class FileRequirement(object): arg_dict["name"] = name elif link.egg_fragment: arg_dict["name"] = link.egg_fragment - if req: - arg_dict["req"] = req - created = cls(**arg_dict) - return created + return cls.create(**arg_dict) @classmethod def from_pipfile(cls, name, pipfile): @@ -466,9 +566,6 @@ class FileRequirement(object): if not uri: uri = pip_shims.shims.path_to_url(path) link = create_link(uri) - req = None - if link.is_artifact and not link.is_wheel and not link.scheme.startswith("file"): - req = init_requirement("{0}@{1}".format(name, uri)) arg_dict = { "name": name, "path": path, @@ -477,13 +574,14 @@ class FileRequirement(object): "link": link, "uri_scheme": uri_scheme, } - if req: - arg_dict["req"] = req - return cls(**arg_dict) + if link.scheme != "file" and not pipfile.get("editable", False): + arg_dict["line"] = "{0}@ {1}".format(name, link.url_without_fragment) + return cls.create(**arg_dict) @property def line_part(self): if self._uri_scheme and self._uri_scheme == "path": + # We may need any one of these for passing to pip seed = self.path or unquote(self.link.url_without_fragment) or self.uri elif (self._uri_scheme and self._uri_scheme == "file") or ( (self.link.is_artifact or self.link.is_wheel) and self.link.url @@ -491,16 +589,16 @@ class FileRequirement(object): seed = unquote(self.link.url_without_fragment) or self.uri # add egg fragments to remote artifacts (valid urls only) if not self._has_hashed_name and self.is_remote_artifact: - if not self.link.is_wheel and self.link.is_artifact: - seed = "{0}@{1}".format(self.name, seed) - else: - seed += "#egg={0}".format(self.name) + seed += "#egg={0}".format(self.name) editable = "-e " if self.editable else "" return "{0}{1}".format(editable, seed) @property def pipfile_part(self): - excludes = ["_base_line", "_has_hashed_name", "setup_path"] + excludes = [ + "_base_line", "_has_hashed_name", "setup_path", "pyproject_path", + "pyproject_requires", "pyproject_backend", "setup_info" + ] filter_func = lambda k, v: bool(v) is True and k.name not in excludes pipfile_dict = attr.asdict(self, filter=filter_func).copy() name = pipfile_dict.pop("name") @@ -687,10 +785,19 @@ class VCSRequirement(FileRequirement): ) if not self.is_local: vcsrepo.obtain() + pyproject_info = None if self.subdirectory: self.setup_path = os.path.join(checkout_dir, self.subdirectory, "setup.py") + self.pyproject_path = os.path.join(checkout_dir, self.subdirectory, "pyproject.toml") + pyproject_info = get_pyproject(os.path.join(checkout_dir, self.subdirectory)) else: self.setup_path = os.path.join(checkout_dir, "setup.py") + self.pyproject_path = os.path.join(checkout_dir, "pyproject.toml") + pyproject_info = get_pyproject(checkout_dir) + if pyproject_info is not None: + pyproject_requires, pyproject_backend = pyproject_info + self.pyproject_requires = pyproject_requires + self.pyproject_backend = pyproject_backend return vcsrepo def get_commit_hash(self): @@ -846,7 +953,10 @@ class VCSRequirement(FileRequirement): @property def pipfile_part(self): - excludes = ["_repo", "_base_line", "setup_path", "_has_hashed_name"] + excludes = [ + "_repo", "_base_line", "setup_path", "_has_hashed_name", "pyproject_path", + "pyproject_requires", "pyproject_backend", "setup_info" + ] filter_func = lambda k, v: bool(v) is True and k.name not in excludes pipfile_dict = attr.asdict(self, filter=filter_func).copy() if "vcs" in pipfile_dict: @@ -952,7 +1062,6 @@ class Requirement(object): line = line.split(" ", 1)[1] if editable else line line, markers = split_markers_from_line(line) line, extras = pip_shims.shims._strip_extras(line) - specifiers = "" if extras: extras = parse_extras(extras) line = line.strip('"').strip("'").strip() @@ -984,7 +1093,6 @@ class Requirement(object): spec_idx = min((line.index(match) for match in spec_matches)) name = line[:spec_idx] version = line[spec_idx:] - specifiers = version if not extras: name, extras = pip_shims.shims._strip_extras(name) if extras: @@ -995,7 +1103,7 @@ class Requirement(object): req_markers = None if markers: req_markers = PackagingRequirement("fakepkg; {0}".format(markers)) - r.req.marker = getattr(req_markers, "marker", None) + r.req.marker = getattr(req_markers, "marker", None) if req_markers else None r.req.local_file = getattr(r.req, "local_file", False) name = getattr(r.req, "name", None) if not name: @@ -1021,7 +1129,15 @@ class Requirement(object): args["extras"] = sorted(dedup([extra.lower() for extra in r.extras])) if hashes: args["hashes"] = hashes - return cls(**args) + cls_inst = cls(**args) + if not cls_inst.is_named and (not cls_inst.editable or cls_inst.req._has_hashed_name): + old_name = cls_inst.req.req.name or cls_inst.req.name + info_dict = cls_inst.run_requires() + calced_name = info_dict.get("name", old_name) + if old_name != calced_name: + cls_inst.req.req.line.replace(old_name, calced_name) + cls_inst.name = cls_inst.req.name = calced_name + return cls_inst @classmethod def from_ireq(cls, ireq): @@ -1074,6 +1190,22 @@ class Requirement(object): cls_inst = cls(**args) if cls_inst.is_named: cls_inst.req.req.line = cls_inst.as_line() + old_name = cls_inst.req.req.name or cls_inst.req.name + if not cls_inst.is_named and not cls_inst.editable and not name: + if cls_inst.is_vcs: + import pip_shims.shims + ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) + info = SetupInfo.from_ireq(ireq) + if info is not None: + info_dict = info.as_dict() + cls_inst.req.setup_info = info + else: + info_dict = {} + else: + info_dict = cls_inst.run_requires() + found_name = info_dict.get("name", old_name) + if old_name != found_name: + cls_inst.req.req.line.replace(old_name, found_name) return cls_inst def as_line( @@ -1159,6 +1291,10 @@ class Requirement(object): def constraint_line(self): return self.as_line() + @property + def is_direct_url(self): + return self.is_file_or_url and self.req.is_direct_url + def as_pipfile(self): good_keys = ( "hashes", @@ -1294,6 +1430,26 @@ class Requirement(object): finder = get_finder(sources=sources) return find_all_matches(finder, self.as_ireq()) + def run_requires(self, sources=None, finder=None): + if self.req and self.req.setup_info is not None: + info_dict = self.req.setup_info.as_dict() + else: + from .setup_info import SetupInfo + if not finder: + from .dependencies import get_finder + finder = get_finder(sources=sources) + info = SetupInfo.from_requirement(self, finder=finder) + if info is None: + return {} + info_dict = info.get_info() + if self.req and not self.req.setup_info: + self.req.setup_info = info + if self.req._has_hashed_name and info_dict.get("name"): + self.req.name = self.name = info_dict["name"] + if self.req.req.name != info_dict["name"]: + self.req.req.name = info_dict["name"] + return info_dict + def merge_markers(self, markers): if not isinstance(markers, Marker): markers = Marker(markers) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py new file mode 100644 index 00000000..319dd6bd --- /dev/null +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -0,0 +1,378 @@ +# -*- coding=utf-8 -*- +import configparser +import contextlib +import os + +import attr +import packaging.version +import packaging.specifiers +import packaging.utils + +try: + from setuptools.dist import distutils +except ImportError: + import distutils + +from appdirs import user_cache_dir +from six.moves.urllib.parse import unquote +from vistir.compat import Path +from vistir.contextmanagers import cd +from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p + +from .utils import init_requirement, get_pyproject + +try: + from os import scandir +except ImportError: + from scandir import scandir + + +CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) + + +@contextlib.contextmanager +def _suppress_distutils_logs(): + """Hack to hide noise generated by `setup.py develop`. + + There isn't a good way to suppress them now, so let's monky-patch. + See https://bugs.python.org/issue25392. + """ + + f = distutils.log.Log._log + + def _log(log, level, msg, args): + if level >= distutils.log.ERROR: + f(log, level, msg, args) + + distutils.log.Log._log = _log + yield + distutils.log.Log._log = f + + +@ensure_mkdir_p(mode=0o775) +def _get_src_dir(): + src = os.environ.get("PIP_SRC") + if src: + return src + virtual_env = os.environ.get("VIRTUAL_ENV") + if virtual_env: + return os.path.join(virtual_env, "src") + return os.path.join(os.getcwd(), "src") # Match pip's behavior. + + +def _prepare_wheel_building_kwargs(ireq): + download_dir = os.path.join(CACHE_DIR, "pkgs") + mkdir_p(download_dir) + + wheel_download_dir = os.path.join(CACHE_DIR, "wheels") + mkdir_p(wheel_download_dir) + + if ireq.source_dir is not None: + src_dir = ireq.source_dir + elif ireq.editable: + src_dir = _get_src_dir() + else: + src_dir = create_tracked_tempdir(prefix="reqlib-src") + + # This logic matches pip's behavior, although I don't fully understand the + # intention. I guess the idea is to build editables in-place, otherwise out + # of the source tree? + if ireq.editable: + build_dir = src_dir + else: + build_dir = create_tracked_tempdir(prefix="reqlib-build") + + return { + "build_dir": build_dir, + "src_dir": src_dir, + "download_dir": download_dir, + "wheel_download_dir": wheel_download_dir, + } + + +def iter_egginfos(path, pkg_name=None): + for entry in scandir(path): + if entry.is_dir(): + if not entry.name.endswith("egg-info"): + for dir_entry in iter_egginfos(entry.path, pkg_name=pkg_name): + yield dir_entry + elif pkg_name is None or entry.name.startswith(pkg_name): + yield entry + + +def find_egginfo(target, pkg_name=None): + egg_dirs = (egg_dir for egg_dir in iter_egginfos(target, pkg_name=pkg_name)) + if pkg_name: + return next(iter(egg_dirs), None) + else: + for egg_dir in egg_dirs: + yield egg_dir + + +def get_metadata(path, pkg_name=None): + if pkg_name: + pkg_name = packaging.utils.canonicalize_name(pkg_name) + egg_dir = next(iter(find_egginfo(path, pkg_name=pkg_name)), None) + if egg_dir is not None: + import pkg_resources + + egg_dir = os.path.abspath(egg_dir) + base_dir = os.path.dirname(egg_dir) + path_metadata = pkg_resources.PathMetadata(base_dir, egg_dir) + dist = next( + iter(pkg_resources.distributions_from_metadata(path_metadata.egg_info)), + None, + ) + if dist: + requires = dist.requires() + dep_map = dist._build_dep_map() + deps = [] + for k in dep_map.keys(): + if k is None: + deps.extend(dep_map.get(k)) + continue + else: + _deps = dep_map.get(k) + k = k.replace(":", "; ") + _deps = [ + pkg_resources.Requirement.parse("{0}{1}".format(str(req), k)) + for req in _deps + ] + deps.extend(_deps) + return { + "name": dist.project_name, + "version": dist.version, + "requires": requires, + } + + +@attr.s(slots=True) +class SetupInfo(object): + name = attr.ib(type=str, default=None) + base_dir = attr.ib(type=Path, default=None) + version = attr.ib(type=packaging.version.Version, default=None) + extras = attr.ib(type=list, default=attr.Factory(list)) + requires = attr.ib(type=dict, default=attr.Factory(dict)) + build_requires = attr.ib(type=list, default=attr.Factory(list)) + build_backend = attr.ib(type=list, default=attr.Factory(list)) + setup_requires = attr.ib(type=dict, default=attr.Factory(list)) + python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None) + extras = attr.ib(type=dict, default=attr.Factory(dict)) + setup_cfg = attr.ib(type=Path, default=None) + setup_py = attr.ib(type=Path, default=None) + pyproject = attr.ib(type=Path, default=None) + ireq = attr.ib(default=None) + extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict) + + def parse_setup_cfg(self): + if self.setup_cfg is not None and self.setup_cfg.exists(): + default_opts = { + "metadata": {"name": "", "version": ""}, + "options": { + "install_requires": "", + "python_requires": "", + "build_requires": "", + "setup_requires": "", + "extras": "", + }, + } + parser = configparser.ConfigParser(default_opts) + parser.read(self.setup_cfg.as_posix()) + if parser.has_option("metadata", "name"): + name = parser.get("metadata", "name") + if not self.name and name is not None: + self.name = name + if parser.has_option("metadata", "version"): + version = parser.get("metadata", "version") + if not self.version and version is not None: + self.version = version + if parser.has_option("options", "install_requires"): + self.requires.update( + { + dep.strip(): init_requirement(dep.strip()) + for dep in parser.get("options", "install_requires").split("\n") + if dep + } + ) + if parser.has_option("options", "python_requires"): + python_requires = parser.get("options", "python_requires") + if python_requires and not self.python_requires: + self.python_requires = python_requires + if parser.has_option("options", "extras_require"): + self.extras.update( + { + section: [ + dep.strip() + for dep in parser.get( + "options.extras_require", section + ).split("\n") + if dep + ] + for section in parser.options("options.extras_require") + } + ) + + def run_setup(self): + if self.setup_py is not None and self.setup_py.exists(): + with cd(self.setup_py.parent), _suppress_distutils_logs(): + from setuptools.dist import distutils + + dist = distutils.core.run_setup( + self.setup_py.as_posix(), ["egg_info", "--egg-base", self.base_dir] + ) + name = dist.get_name() + if name: + self.name = name + if dist.python_requires and not self.python_requires: + self.python_requires = packaging.specifiers.SpecifierSet( + dist.python_requires + ) + if dist.extras_require and not self.extras: + self.extras = dist.extras_require + install_requires = dist.get_requires() + if not install_requires: + install_requires = dist.install_requires + if install_requires and not self.requires: + requirements = [init_requirement(req) for req in install_requires] + self.requires.update({req.key: req for req in requirements}) + if dist.setup_requires and not self.setup_requires: + self.setup_requires = dist.setup_requires + if not self.version: + self.version = dist.get_version() + + def get_egg_metadata(self): + if self.setup_py is not None and self.setup_py.exists(): + metadata = get_metadata(self.setup_py.parent.as_posix(), pkg_name=self.name) + if metadata: + if not self.name: + self.name = metadata.get("name", self.name) + if not self.version: + self.version = metadata.get("version", self.version) + self.requires.update( + {req.key: req for req in metadata.get("requires", {})} + ) + + def run_pyproject(self): + if self.pyproject and self.pyproject.exists(): + result = get_pyproject(self.pyproject.parent) + if result is not None: + requires, backend = result + if backend: + self.build_backend = backend + if requires and not self.build_requires: + self.build_requires = requires + + def get_info(self): + if self.setup_cfg and self.setup_cfg.exists(): + self.parse_setup_cfg() + if self.setup_py and self.setup_py.exists(): + if not self.requires or not self.name: + try: + self.run_setup() + except Exception as e: + self.get_egg_metadata() + if not self.requires or not self.name: + self.get_egg_metadata() + + if self.pyproject and self.pyproject.exists(): + self.run_pyproject() + return self.as_dict() + + def as_dict(self): + prop_dict = { + "name": self.name, + "version": self.version, + "base_dir": self.base_dir, + "ireq": self.ireq, + "build_backend": self.build_backend, + "build_requires": self.build_requires, + "requires": self.requires, + "setup_requires": self.setup_requires, + "python_requires": self.python_requires, + "extras": self.extras, + "extra_kwargs": self.extra_kwargs, + "setup_cfg": self.setup_cfg, + "setup_py": self.setup_py, + "pyproject": self.pyproject, + } + return {k: v for k, v in prop_dict.items() if v} + + @classmethod + def from_requirement(cls, requirement, finder=None): + ireq = requirement.as_ireq() + subdir = getattr(requirement.req, "subdirectory", None) + return cls.from_ireq(ireq, subdir=subdir, finder=finder) + + @classmethod + def from_ireq(cls, ireq, subdir=None, finder=None): + import pip_shims.shims + + if ireq.link.is_wheel: + return + if not finder: + from .dependencies import get_finder + + finder = get_finder() + kwargs = _prepare_wheel_building_kwargs(ireq) + ireq.populate_link(finder, False, False) + ireq.ensure_has_source_dir(kwargs["build_dir"]) + if not ( + ireq.editable + and pip_shims.shims.is_file_url(ireq.link) + and not ireq.link.is_artifact + ): + if ireq.is_wheel: + only_download = True + download_dir = kwargs["wheel_download_dir"] + else: + only_download = False + download_dir = kwargs["download_dir"] + ireq_src_dir = None + if ireq.link.scheme == "file": + path = pip_shims.shims.url_to_path(unquote(ireq.link.url_without_fragment)) + if pip_shims.shims.is_installable_dir(path): + ireq_src_dir = path + if not ireq.editable or not (pip_shims.is_file_url(ireq.link) and ireq_src_dir): + pip_shims.shims.unpack_url( + ireq.link, + ireq.source_dir, + download_dir, + only_download=only_download, + session=finder.session, + hashes=ireq.hashes(False), + progress_bar="off", + ) + if ireq.editable: + created = cls.create( + ireq.source_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs + ) + else: + build_dir = ireq.build_location(kwargs["build_dir"]) + ireq._temp_build_dir.path = kwargs["build_dir"] + created = cls.create( + build_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs + ) + created.get_info() + return created + + @classmethod + def create(cls, base_dir, subdirectory=None, ireq=None, kwargs=None): + if not base_dir or base_dir is None: + return + + creation_kwargs = {"extra_kwargs": kwargs} + if not isinstance(base_dir, Path): + base_dir = Path(base_dir) + creation_kwargs["base_dir"] = base_dir.as_posix() + pyproject = base_dir.joinpath("pyproject.toml") + + if subdirectory is not None: + base_dir = base_dir.joinpath(subdirectory) + setup_py = base_dir.joinpath("setup.py") + setup_cfg = base_dir.joinpath("setup.cfg") + creation_kwargs["pyproject"] = pyproject + creation_kwargs["setup_py"] = setup_py + creation_kwargs["setup_cfg"] = setup_cfg + if ireq: + creation_kwargs["ireq"] = ireq + return cls(**creation_kwargs) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index aa7ffd68..2b47ee9b 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import io import os import sys @@ -9,6 +10,7 @@ from itertools import chain, groupby from operator import attrgetter import six +import tomlkit from attr import validators from first import first @@ -17,7 +19,7 @@ from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet from vistir.misc import dedup -from ..utils import SCHEME_LIST, VCS_LIST, is_star, strip_ssh_from_git_uri, add_ssh_scheme_to_git_uri +from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri HASH_STRING = " --hash={0}" @@ -93,6 +95,7 @@ def build_vcs_link(vcs, uri, name=None, ref=None, subdirectory=None, extras=None if extras: extras = extras_to_string(extras) uri = "{0}{1}".format(uri, extras) + # if subdirectory: if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) return create_link(uri) @@ -112,6 +115,42 @@ def get_version(pipfile_entry): return "" +def get_pyproject(path): + from vistir.compat import Path + if not path: + return + if not isinstance(path, Path): + path = Path(path) + if not path.is_dir(): + path = path.parent + pp_toml = path.joinpath("pyproject.toml") + setup_py = path.joinpath("setup.py") + if not pp_toml.exists(): + if setup_py.exists(): + return None + else: + pyproject_data = {} + with io.open(pp_toml.as_posix(), encoding="utf-8") as fh: + pyproject_data = tomlkit.loads(fh.read()) + build_system = pyproject_data.get("build-system", None) + if build_system is None: + if setup_py.exists(): + requires = ["setuptools", "wheel"] + backend = "setuptools.build_meta" + else: + requires = ["setuptools>=38.2.5", "wheel"] + backend = "setuptools.build_meta" + build_system = { + "requires": requires, + "build-backend": backend + } + pyproject_data["build_system"] = build_system + else: + requires = build_system.get("requires") + backend = build_system.get("build-backend") + return (requires, backend) + + def split_markers_from_line(line): """Split markers from a dependency""" if not any(line.startswith(uri_prefix) for uri_prefix in SCHEME_LIST): diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index dd8cc3a4..6a15db3f 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -4,7 +4,6 @@ import os import pip_shims - @attr.s class VCSRepository(object): url = attr.ib() @@ -32,7 +31,7 @@ class VCSRepository(object): def obtain(self): if (os.path.exists(self.checkout_directory) and not - self.repo_instance.is_repository_directory(self.checkout_directory)): + self.repo_instance.is_repository_directory(self.checkout_directory)): self.repo_instance.unpack(self.checkout_directory) elif not os.path.exists(self.checkout_directory): self.repo_instance.obtain(self.checkout_directory) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0ab0ab22..6ed95b3e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -56,8 +56,17 @@ def check_github_ssh(): return res +def check_for_mercurial(): + c = delegator.run("hg --help") + if c.return_code != 0: + return False + else: + return True + + TESTS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PYPI_VENDOR_DIR = os.path.join(TESTS_ROOT, 'pypi') +WE_HAVE_HG = check_for_mercurial() prepare_pypi_packages(PYPI_VENDOR_DIR) @@ -66,6 +75,8 @@ def pytest_runtest_setup(item): pytest.skip('requires internet') if item.get_marker('needs_github_ssh') is not None and not WE_HAVE_GITHUB_SSH_KEYS: pytest.skip('requires github ssh') + if item.get_marker('needs_hg') is not None and not WE_HAVE_HG: + pytest.skip('requires mercurial') @pytest.fixture @@ -100,6 +111,8 @@ def isolate(pathlib_tmpdir): os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org") mkdir_p(os.path.join(home_dir, ".virtualenvs")) os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) + global WE_HAVE_GITHUB_SSH_KEYS + WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() WE_HAVE_INTERNET = check_internet() diff --git a/tests/integration/test_uninstall.py b/tests/integration/test_uninstall.py index e19a1400..5f493cac 100644 --- a/tests/integration/test_uninstall.py +++ b/tests/integration/test_uninstall.py @@ -84,7 +84,7 @@ def test_uninstall_all_local_files(PipenvInstance, testsroot): # Not sure where travis/appveyor run tests from source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) - with PipenvInstance() as p: + with PipenvInstance(chdir=True) as p: shutil.copy(source_path, os.path.join(p.path, file_name)) os.mkdir(os.path.join(p.path, "requests")) c = p.pipenv("install {}".format(file_name)) @@ -92,7 +92,9 @@ def test_uninstall_all_local_files(PipenvInstance, testsroot): c = p.pipenv("uninstall --all") assert c.return_code == 0 assert "requests" in c.out - assert "requests" not in p.pipfile["packages"] + # Uninstall --all is not supposed to remove things from the pipfile + # Note that it didn't before, but that instead local filenames showed as hashes + assert "requests" in p.pipfile["packages"] @pytest.mark.run From dec7be54d716d078390232c5ac5a78991ca3e0b1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 16:39:23 -0500 Subject: [PATCH 142/218] Introduce `pipenv.environments.Environment` - Specific construct for isolationg operations Signed-off-by: Dan Ryan --- pipenv/core.py | 107 ++++---- pipenv/environment.py | 618 ++++++++++++++++++++++++++++++++++++++++++ pipenv/project.py | 176 +++--------- 3 files changed, 713 insertions(+), 188 deletions(-) create mode 100644 pipenv/environment.py diff --git a/pipenv/core.py b/pipenv/core.py index 3cbd1645..021b0f3c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -915,7 +915,15 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): project_file_name = os.path.join(project.virtualenv_location, ".project") with open(project_file_name, "w") as f: f.write(vistir.misc.fs_str(project.project_directory)) - fix_venv_site(project.env_paths["lib"]) + from .environment import Environment + sources = project.pipfile_sources + project._environment = Environment( + prefix=project.get_location_for_virtualenv(), + is_venv=True, + sources=sources, + pipfile=project.parsed_pipfile + ) + project._environment.add_dist("pipenv") # Say where the virtualenv is. do_where(virtualenv=True, bare=False) @@ -1129,7 +1137,7 @@ def do_purge(bare=False, downloads=False, allow_global=False): # Remove comments from the output, if any. installed = set([ - pep423_name(pkg.project_name) for pkg in project.get_installed_packages() + pep423_name(pkg.project_name) for pkg in project.environment.get_installed_packages() ]) bad_pkgs = set([pep423_name(pkg) for pkg in BAD_PACKAGES]) # Remove setuptools, pip, etc from targets for removal @@ -1662,7 +1670,7 @@ def do_outdated(pypi_mirror=None): packages = {} package_info = namedtuple("PackageInfo", ["name", "installed", "available"]) - installed_packages = project.get_installed_packages() + installed_packages = project.environment.get_installed_packages() outdated_packages = { canonicalize_name(pkg.project_name): package_info (pkg.project_name, pkg.parsed_version, pkg.latest_version) @@ -1916,7 +1924,14 @@ def do_install( # make a tuple of (display_name, entry) pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages] - + if not system and not project.virtualenv_exists: + with create_spinner("Creating virtualenv...") as sp: + try: + do_create_virtualenv(pypi_mirror=pypi_mirror) + except KeyboardInterrupt: + cleanup_virtualenv(bare=(not environments.is_verbose())) + sys.exit(1) + sp.write_err("Ok...") for pkg_line in pkg_list: click.echo( crayons.normal( @@ -1925,8 +1940,7 @@ def do_install( ) ) # pip install: - with vistir.contextmanagers.temp_environ(), \ - create_spinner("Installing...") as sp: + with vistir.contextmanagers.temp_environ(), create_spinner("Installing...") as sp: os.environ["PIP_USER"] = vistir.compat.fs_str("0") try: pkg_requirement = Requirement.from_line(pkg_line) @@ -2055,30 +2069,17 @@ def do_uninstall( package_map = { canonicalize_name(p): p for p in packages if p } - installed_package_names = set([ - canonicalize_name(pkg.project_name) for pkg in project.get_installed_packages() - ]) + installed_package_names = project.installed_package_names # Intelligently detect if --dev should be used or not. lockfile_packages = set() if project.lockfile_exists: - develop = set( - [canonicalize_name(k) for k in project.lockfile_content["develop"].keys()] - ) - default = set( - [canonicalize_name(k) for k in project.lockfile_content["default"].keys()] - ) - lockfile_packages |= develop | default + project_pkg_names = project.lockfile_package_names else: - develop = set( - [canonicalize_name(k) for k in project.dev_packages.keys()] - ) - default = set( - [canonicalize_name(k) for k in project.packages.keys()] - ) + project_pkg_names = project.pipfile_package_names pipfile_remove = True # Uninstall [dev-packages], if --dev was provided. if all_dev: - if "dev-packages" not in project.parsed_pipfile and not develop: + if "dev-packages" not in project.parsed_pipfile and not project_pkg_names["dev"]: click.echo( crayons.normal( "No {0} to uninstall.".format(crayons.red("[dev-packages]")), @@ -2091,28 +2092,33 @@ def do_uninstall( fix_utf8("Un-installing {0}…".format(crayons.red("[dev-packages]"))), bold=True ) ) - package_names = develop + package_names = project_pkg_names["dev"] + # Remove known "bad packages" from the list. - bad_pkgs = set([canonicalize_name(pkg) for pkg in BAD_PACKAGES]) - for bad_package in BAD_PACKAGES: - normalized_bad_pkg = canonicalize_name(bad_package) - if normalized_bad_pkg in package_map: - if environments.is_verbose(): - click.echo("Ignoring {0}.".format(bad_package), err=True) - pkg_name_index = package_names.index(package_map[normalized_bad_pkg]) - del package_names[pkg_name_index] - used_packages = develop | default & installed_package_names + bad_pkgs = get_canonical_names(BAD_PACKAGES) + ignored_packages = bad_pkgs & set(list(package_map.keys())) + for ignored_pkg in ignored_packages: + if environments.is_verbose(): + click.echo("Ignoring {0}.".format(ignored_pkg), err=True) + pkg_name_index = package_names.index(package_map[ignored_pkg]) + del package_names[pkg_name_index] + + used_packages = project_pkg_names["combined"] & installed_package_names failure = False packages_to_remove = set() if all: - package_names = develop | default click.echo( - crayons.normal(fix_utf8("Un-installing all packages from virtualenv…"), bold=True) + crayons.normal( + fix_utf8("Un-installing all {0} and {1}…".format( + crayons.red("[dev-packages]"), + crayons.red("[packages]"), + )), bold=True + ) ) - do_purge(allow_global=system) - return + do_purge(bare=False, allow_global=system) + sys.exit(0) if all_dev: - package_names = develop + package_names = project_pkg_names["dev"] else: package_names = set([pkg_name for pkg_name in package_names]) selected_pkg_map = { @@ -2120,7 +2126,7 @@ def do_uninstall( } packages_to_remove = [ p for normalized, p in selected_pkg_map.items() - if (normalized in used_packages and normalized not in bad_pkgs) + if normalized in (used_packages - bad_pkgs) ] for normalized, package_name in selected_pkg_map.items(): click.echo( @@ -2130,15 +2136,16 @@ def do_uninstall( ) # Uninstall the package. if package_name in packages_to_remove: - cmd = "{0} uninstall {1} -y".format( - escape_grouped_arguments(which_pip(allow_global=system)), package_name, - ) - if environments.is_verbose(): - click.echo("$ {0}".format(cmd)) - c = delegator.run(cmd) - click.echo(crayons.blue(c.out)) - if c.return_code != 0: - failure = True + with project.environment.activated(): + cmd = "{0} uninstall {1} -y".format( + escape_grouped_arguments(which_pip(allow_global=system)), package_name, + ) + if environments.is_verbose(): + click.echo("$ {0}".format(cmd)) + c = delegator.run(cmd) + click.echo(crayons.blue(c.out)) + if c.return_code != 0: + failure = True if not failure and pipfile_remove: in_packages = project.get_package_name_in_pipfile(package_name, dev=False) in_dev_packages = project.get_package_name_in_pipfile( @@ -2646,9 +2653,9 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro ensure_lockfile(pypi_mirror=pypi_mirror) # Make sure that the virtualenv's site packages are configured correctly # otherwise we may end up removing from the global site packages directory - fix_venv_site(project.env_paths["lib"]) installed_package_names = [ - canonicalize_name(pkg.project_name) for pkg in project.get_installed_packages() + canonicalize_name(pkg.project_name) for pkg + in project.environment.get_installed_packages() ] # Remove known "bad packages" from the list. for bad_package in BAD_PACKAGES: diff --git a/pipenv/environment.py b/pipenv/environment.py new file mode 100644 index 00000000..db0e22aa --- /dev/null +++ b/pipenv/environment.py @@ -0,0 +1,618 @@ +# -*- coding=utf-8 -*- + +import contextlib +import importlib +import json +import os +import sys +import operator +import pkg_resources +import six + +from distutils.sysconfig import get_python_lib +from sysconfig import get_paths + +from cached_property import cached_property + +import vistir +import pipenv + +BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path) + + +class Environment(object): + def __init__(self, prefix=None, is_venv=False, base_working_set=None, pipfile=None, sources=None): + super(Environment, self).__init__() + self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} + self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET + self.is_venv = not os.path.samefile(os.path.abspath(prefix), sys.prefix) + if not sources: + sources = [] + self.sources = sources + self.extra_dists = [] + prefix = prefix if prefix else sys.prefix + self.prefix = vistir.compat.Path(prefix) + + def safe_import(self, name): + """Helper utility for reimporting previously imported modules while inside the env""" + module = None + if name not in self._modules: + self._modules[name] = importlib.import_module(name) + module = self._modules[name] + if not module: + dist = next(iter( + dist for dist in self.base_working_set if dist.project_name == name + ), None) + if dist: + dist.activate() + module = importlib.import_module(name) + if name in sys.modules: + try: + six.moves.reload_module(module) + six.moves.reload_module(sys.modules[name]) + except TypeError: + del sys.modules[name] + sys.modules[name] = self._modules[name] + return module + + @classmethod + def resolve_dist(cls, dist, working_set): + """Given a local distribution and a working set, returns all dependencies from the set. + + :param dist: A single distribution to find the dependencies of + :type dist: :class:`pkg_resources.Distribution` + :param working_set: A working set to search for all packages + :type working_set: :class:`pkg_resources.WorkingSet` + :return: A set of distributions which the package depends on, including the package + :rtype: set(:class:`pkg_resources.Distribution`) + """ + + deps = set() + deps.add(dist) + try: + reqs = dist.requires() + except AttributeError: + return deps + for req in reqs: + dist = working_set.find(req) + deps |= cls.resolve_dist(dist, working_set) + return deps + + def add_dist(self, dist_name): + dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + extras = self.resolve_dist(dist, self.base_working_set) + if extras: + self.extra_dists.extend(extras) + + @cached_property + def python_version(self): + with self.activated(): + from sysconfig import get_python_version + py_version = get_python_version() + return py_version + + @property + def python_info(self): + include_dir = self.prefix / "include" + python_path = next(iter(list(include_dir.iterdir())), None) + if python_path and python_path.name.startswith("python"): + python_version = python_path.name.replace("python", "") + py_version_short, abiflags = python_version[:3], python_version[3:] + return {"py_version_short": py_version_short, "abiflags": abiflags} + return {} + + @cached_property + def base_paths(self): + """ + Returns the context appropriate paths for the environment. + + :return: A dictionary of environment specific paths to be used for installation operations + :rtype: dict + + .. note:: The implementation of this is borrowed from a combination of pip and + virtualenv and is likely to change at some point in the future. + + >>> from pipenv.core import project + >>> from pipenv.environment import Environment + >>> env = Environment(prefix=project.virtualenv_location, is_venv=True, sources=project.sources) + >>> import pprint + >>> pprint.pprint(env.base_paths) + {'PATH': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/bin::/bin:/usr/bin', + 'PYTHONPATH': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'data': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW', + 'include': '/home/hawk/.pyenv/versions/3.7.1/include/python3.7m', + 'libdir': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'platinclude': '/home/hawk/.pyenv/versions/3.7.1/include/python3.7m', + 'platlib': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'platstdlib': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7', + 'prefix': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW', + 'purelib': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'scripts': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/bin', + 'stdlib': '/home/hawk/.pyenv/versions/3.7.1/lib/python3.7'} + """ + + prefix = self.prefix.as_posix() + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + paths = get_paths(install_scheme, vars={ + 'base': prefix, + 'platbase': prefix, + }) + paths["PATH"] = paths["scripts"] + os.pathsep + os.defpath + if "prefix" not in paths: + paths["prefix"] = prefix + purelib = get_python_lib(plat_specific=0, prefix=prefix) + platlib = get_python_lib(plat_specific=1, prefix=prefix) + if purelib == platlib: + lib_dirs = purelib + else: + lib_dirs = purelib + os.pathsep + platlib + paths["libdir"] = purelib + paths["purelib"] = purelib + paths["platlib"] = platlib + paths['PYTHONPATH'] = lib_dirs + paths["libdirs"] = lib_dirs + return paths + + @cached_property + def script_basedir(self): + """Path to the environment scripts dir""" + script_dir = self.base_paths["scripts"] + return script_dir + + @property + def python(self): + """Path to the environment python""" + py = vistir.compat.Path(self.base_paths["scripts"]).joinpath("python").as_posix() + if not py: + return vistir.compat.Path(sys.executable).as_posix() + return py + + @cached_property + def sys_path(self): + """The system path inside the environment + + :return: The :data:`sys.path` from the environment + :rtype: list + """ + + current_executable = vistir.compat.Path(sys.executable).as_posix() + if not self.python or self.python == current_executable: + return sys.path + elif any([sys.prefix == self.prefix, not self.is_venv]): + return sys.path + cmd_args = [self.python, "-c", "import json, sys; print(json.dumps(sys.path))"] + path, _ = vistir.misc.run(cmd_args, return_object=False, nospin=True, block=True, combine_stderr=False) + path = json.loads(path.strip()) + return path + + @cached_property + def system_paths(self): + paths = {} + paths = get_paths() + return paths + + @cached_property + def sys_prefix(self): + """The prefix run inside the context of the environment + + :return: The python prefix inside the environment + :rtype: :data:`sys.prefix` + """ + + command = [self.python, "-c" "import sys; print(sys.prefix)"] + c = vistir.misc.run(command, return_object=True, block=True, nospin=True) + sys_prefix = vistir.compat.Path(vistir.misc.to_text(c.out).strip()).as_posix() + return sys_prefix + + @cached_property + def paths(self): + paths = {} + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + paths = self.base_paths + os.environ["PATH"] = paths["PATH"] + os.environ["PYTHONPATH"] = paths["PYTHONPATH"] + if "headers" not in paths: + paths["headers"] = paths["include"] + return paths + + @property + def scripts_dir(self): + return self.paths["scripts"] + + @property + def libdir(self): + purelib = self.paths.get("purelib", None) + if purelib and os.path.exists(purelib): + return "purelib", purelib + return "platlib", self.paths["platlib"] + + def get_distributions(self): + """Retrives the distributions installed on the library path of the environment + + :return: A set of distributions found on the library path + :rtype: iterator + """ + + pkg_resources = self.safe_import("pkg_resources") + return pkg_resources.find_distributions(self.paths["PYTHONPATH"]) + + def find_egg(self, egg_dist): + import site + site_packages = get_python_lib() + search_filename = "{0}.egg-link".format(egg_dist.project_name) + try: + user_site = site.getusersitepackages() + except AttributeError: + user_site = site.USER_SITE + search_locations = [site_packages, user_site] + for site_directory in search_locations: + egg = os.path.join(site_directory, search_filename) + if os.path.isfile(egg): + return egg + + def locate_dist(self, dist): + location = self.find_egg(dist) + if not location: + return dist.location + + def dist_is_in_project(self, dist): + from .project import _normalized + prefix = _normalized(self.base_paths["prefix"]) + location = self.locate_dist(dist) + if not location: + return False + return _normalized(location).startswith(prefix) + + def get_installed_packages(self): + workingset = self.get_working_set() + packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] + return packages + + def get_finder(self): + from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder + from .environments import PIPENV_CACHE_DIR + index_urls = [source.get("url") for source in self.sources] + + class PipCommand(Command): + name = "PipCommand" + + pip_command = PipCommand() + index_opts = cmdoptions.make_option_group( + index_group, pip_command.parser + ) + cmd_opts = pip_command.cmd_opts + pip_command.parser.insert_option_group(0, index_opts) + pip_command.parser.insert_option_group(0, cmd_opts) + pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources, []) + pip_options, _ = pip_command.parser.parse_args(pip_args) + pip_options.cache_dir = PIPENV_CACHE_DIR + pip_options.pre = self.pipfile.get("pre", False) + with pip_command._build_session(pip_options) as session: + finder = PackageFinder( + find_links=pip_options.find_links, + index_urls=index_urls, allow_all_prereleases=pip_options.pre, + trusted_hosts=pip_options.trusted_hosts, + process_dependency_links=pip_options.process_dependency_links, + session=session + ) + yield finder + + def get_package_info(self): + dependency_links = [] + packages = self.get_installed_packages() + # This code is borrowed from pip's current implementation + for dist in packages: + if dist.has_metadata('dependency_links.txt'): + dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) + + with self.get_finder() as finder: + finder.add_dependency_links(dependency_links) + + for dist in packages: + typ = 'unknown' + all_candidates = finder.find_all_candidates(dist.key) + if not finder.pip_options.pre: + # Remove prereleases + all_candidates = [ + candidate for candidate in all_candidates + if not candidate.version.is_prerelease + ] + + if not all_candidates: + continue + best_candidate = max(all_candidates, key=finder._candidate_sort_key) + remote_version = best_candidate.version + if best_candidate.location.is_wheel: + typ = 'wheel' + else: + typ = 'sdist' + # This is dirty but makes the rest of the code much cleaner + dist.latest_version = remote_version + dist.latest_filetype = typ + yield dist + + def get_outdated_packages(self): + return [ + pkg for pkg in self.get_package_info() + if pkg.latest_version._version > pkg.parsed_version._version + ] + + def get_package_requirements(self): + from .vendor.pipdeptree import flatten, sorted_tree, build_dist_index, construct_tree + dist_index = build_dist_index(self.get_installed_packages()) + tree = sorted_tree(construct_tree(dist_index)) + branch_keys = set(r.key for r in flatten(tree.values())) + nodes = [p for p in tree.keys() if p.key not in branch_keys] + key_tree = dict((k.key, v) for k, v in tree.items()) + get_children = lambda n: key_tree.get(n.key, []) + + def aux(node, parent=None, chain=None): + if chain is None: + chain = [node.project_name] + + d = node.as_dict() + if parent: + d['required_version'] = node.version_spec if node.version_spec else 'Any' + else: + d['required_version'] = d['installed_version'] + + d['dependencies'] = [ + aux(c, parent=node, chain=chain+[c.project_name]) + for c in get_children(node) + if c.project_name not in chain + ] + + return d + return [aux(p) for p in nodes] + + def get_working_set(self): + """Retrieve the working set of installed packages for the environment. + + :return: The working set for the environment + :rtype: :class:`pkg_resources.WorkingSet` + """ + + working_set = pkg_resources.WorkingSet(self.sys_path) + return working_set + + def is_installed(self, pkgname): + """Given a package name, returns whether it is installed in the environment + + :param str pkgname: The name of a package + :return: Whether the supplied package is installed in the environment + :rtype: bool + """ + + return any(d for d in self.get_distributions() if d.project_name == pkgname) + + def run(self, cmd, cwd=os.curdir): + """Run a command with :class:`~subprocess.Popen` in the context of the environment + + :param cmd: A command to run in the environment + :type cmd: str or list + :param str cwd: The working directory in which to execute the command, defaults to :data:`os.curdir` + :return: A finished command object + :rtype: :class:`~subprocess.Popen` + """ + + c = None + with self.activated(): + script = vistir.cmdparse.Script.parse(cmd) + c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) + return c + + def run_py(self, cmd, cwd=os.curdir): + """Run a python command in the enviornment context. + + :param cmd: A command to run in the environment - runs with `python -c` + :type cmd: str or list + :param str cwd: The working directory in which to execute the command, defaults to :data:`os.curdir` + :return: A finished command object + :rtype: :class:`~subprocess.Popen` + """ + + c = None + if isinstance(cmd, six.string_types): + script = vistir.cmdparse.Script.parse("{0} -c {1}".format(self.python, cmd)) + else: + script = vistir.cmdparse.Script.parse([self.python, "-c"] + list(cmd)) + with self.activated(): + c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) + return c + + def run_activate_this(self): + """Runs the environment's inline activation script""" + if self.is_venv: + activate_this = os.path.join(self.scripts_dir, "activate_this.py") + if not os.path.isfile(activate_this): + raise OSError("No such file: {0!s}".format(activate_this)) + with open(activate_this, "r") as f: + code = compile(f.read(), activate_this, "exec") + exec(code, dict(__file__=activate_this)) + + @contextlib.contextmanager + def activated(self, include_extras=True, extra_dists=None): + """Helper context manager to activate the environment. + + This context manager will set the following variables for the duration + of its activation: + + * sys.prefix + * sys.path + * os.environ["VIRTUAL_ENV"] + * os.environ["PATH"] + + In addition, it will make any distributions passed into `extra_dists` available + on `sys.path` while inside the context manager, as well as making `passa` itself + available. + + The environment's `prefix` as well as `scripts_dir` properties are both prepended + to `os.environ["PATH"]` to ensure that calls to `~Environment.run()` use the + environment's path preferentially. + """ + + if not extra_dists: + extra_dists = [] + original_path = sys.path + original_prefix = sys.prefix + parent_path = vistir.compat.Path(__file__).absolute().parent + vendor_dir = parent_path.joinpath("vendor").as_posix() + patched_dir = parent_path.joinpath("patched").as_posix() + parent_path = parent_path.as_posix() + prefix = self.prefix.as_posix() + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PATH"] = os.pathsep.join([ + vistir.compat.fs_str(self.scripts_dir), + vistir.compat.fs_str(self.prefix.as_posix()), + os.environ.get("PATH", "") + ]) + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + os.environ["PATH"] = self.base_paths["PATH"] + os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] + if self.is_venv: + os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix) + sys.path = self.sys_path + sys.prefix = self.sys_prefix + site = self.safe_import("site") + site.addsitedir(self.base_paths["purelib"]) + if include_extras: + site.addsitedir(parent_path) + sys.path.extend([parent_path, patched_dir, vendor_dir]) + extra_dists = list(self.extra_dists) + extra_dists + for extra_dist in extra_dists: + if extra_dist not in self.get_working_set(): + extra_dist.activate(self.sys_path) + try: + yield + finally: + sys.path = original_path + sys.prefix = original_prefix + six.moves.reload_module(pkg_resources) + + @cached_property + def finders(self): + from pipenv.vendor.pythonfinder import Finder + finders = [ + Finder(path=self.base_paths["scripts"], global_search=gs, system=False) + for gs in (False, True) + ] + return finders + + @property + def finder(self): + return next(iter(self.finders), None) + + def which(self, search, as_path=True): + find = operator.methodcaller("which", search) + result = next(iter(filter(None, (find(finder) for finder in self.finders))), None) + if not result: + result = self._which(search) + else: + if as_path: + result = str(result.path) + return result + + def get_install_args(self, editable=False, setup_path=None): + install_arg = "install" if not editable else "develop" + install_keys = ["headers", "purelib", "platlib", "scripts", "data"] + install_args = [ + self.environment.python, "-u", "-c", SETUPTOOLS_SHIM % setup_path, + install_arg, "--single-version-externally-managed", "--no-deps", + "--prefix={0}".format(self.base_paths["prefix"]), "--no-warn-script-location" + ] + for key in install_keys: + install_args.append( + "--install-{0}={1}".format(key, self.base_paths[key]) + ) + return install_args + + def install(self, requirements): + if not isinstance(requirements, (tuple, list)): + requirements = [requirements,] + with self.get_finder() as finder: + args = [] + for format_control in ('no_binary', 'only_binary'): + formats = getattr(finder.format_control, format_control) + args.extend(('--' + format_control.replace('_', '-'), + ','.join(sorted(formats or {':none:'})))) + if finder.index_urls: + args.extend(['-i', finder.index_urls[0]]) + for extra_index in finder.index_urls[1:]: + args.extend(['--extra-index-url', extra_index]) + else: + args.append('--no-index') + for link in finder.find_links: + args.extend(['--find-links', link]) + for _, host, _ in finder.secure_origins: + args.extend(['--trusted-host', host]) + if finder.allow_all_prereleases: + args.append('--pre') + if finder.process_dependency_links: + args.append('--process-dependency-links') + args.append('--') + args.extend(requirements) + out, _ = vistir.misc.run(args, return_object=False, nospin=True, block=True, + combine_stderr=False) + + @contextlib.contextmanager + def uninstall(self, pkgname, *args, **kwargs): + """A context manager which allows uninstallation of packages from the environment + + :param str pkgname: The name of a package to uninstall + + >>> env = Environment("/path/to/env/root") + >>> with env.uninstall("pytz", auto_confirm=True, verbose=False) as uninstaller: + cleaned = uninstaller.paths + >>> if cleaned: + print("uninstalled packages: %s" % cleaned) + """ + + auto_confirm = kwargs.pop("auto_confirm", True) + verbose = kwargs.pop("verbose", False) + with self.activated(): + monkey_patch = next(iter( + dist for dist in self.base_working_set + if dist.project_name == "recursive-monkey-patch" + ), None) + if monkey_patch: + monkey_patch.activate() + pip_shims = self.safe_import("pip_shims") + pathset_base = pip_shims.UninstallPathSet + import recursive_monkey_patch + recursive_monkey_patch.monkey_patch( + PatchedUninstaller, pathset_base + ) + dist = next( + iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), + None + ) + pathset = pathset_base.from_dist(dist) + if pathset is not None: + pathset.remove(auto_confirm=auto_confirm, verbose=verbose) + try: + yield pathset + except Exception as e: + if pathset is not None: + pathset.rollback() + else: + if pathset is not None: + pathset.commit() + if pathset is None: + return + + +class PatchedUninstaller(object): + def _permitted(self, path): + return True + + +SETUPTOOLS_SHIM = ( + "import setuptools, tokenize;__file__=%r;" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))" +) diff --git a/pipenv/project.py b/pipenv/project.py index d4713b89..d3b56be6 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -19,6 +19,7 @@ import vistir import toml import tomlkit +from .environment import Environment from .cmdparse import Script from .utils import ( pep423_name, @@ -35,7 +36,7 @@ from .utils import ( get_workon_home, is_virtual_environment, looks_like_dir, - sys_version + get_canonical_names ) from .environments import ( PIPENV_MAX_DEPTH, @@ -45,7 +46,6 @@ from .environments import ( PIPENV_TEST_INDEX, PIPENV_PYTHON, PIPENV_DEFAULT_PYTHON_VERSION, - PIPENV_CACHE_DIR ) @@ -154,6 +154,7 @@ class Project(object): self._lockfile_newlines = DEFAULT_NEWLINES self._requirements_location = None self._original_dir = os.path.abspath(os.curdir) + self._environment = None self._which = which self._build_system = { "requires": ["setuptools", "wheel"] @@ -316,105 +317,48 @@ class Project(object): import pkg_resources return pkg_resources.WorkingSet(sys_path) - def find_egg(self, egg_dist): - import site - from distutils import sysconfig as distutils_sysconfig - site_packages = distutils_sysconfig.get_python_lib() - search_filename = "{0}.egg-link".format(egg_dist.project_name) - try: - user_site = site.getusersitepackages() - except AttributeError: - user_site = site.USER_SITE - search_locations = [site_packages, user_site] - for site_directory in search_locations: - egg = os.path.join(site_directory, search_filename) - if os.path.isfile(egg): - return egg + @property + def installed_packages(self): + return self.environment.get_installed_packages() - def locate_dist(self, dist): - location = self.find_egg(dist) - if not location: - return dist.location + @property + def installed_package_names(self): + return get_canonical_names([pkg.key for pkg in self.installed_packages]) - def dist_is_in_project(self, dist): - prefix = _normalized(self.env_paths["prefix"]) - location = self.locate_dist(dist) - if not location: - return False - return _normalized(location).startswith(prefix) + @property + def lockfile_package_names(self): + dev_keys = get_canonical_names(self.lockfile_content["develop"].keys()) + default_keys = get_canonical_names(self.lockfile_content["default"].keys()) + return { + "dev": dev_keys, + "default": default_keys, + "combined": dev_keys | default_keys + } - def get_installed_packages(self): - workingset = self.working_set - if self.virtualenv_exists: - packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] - else: - packages = [pkg for pkg in packages] - return packages + @property + def pipfile_package_names(self): + dev_keys = get_canonical_names(self.dev_packages.keys()) + default_keys = get_canonical_names(self.packages.keys()) + return { + "dev": dev_keys, + "default": default_keys, + "combined": dev_keys | default_keys + } - def get_package_info(self): - from .utils import prepare_pip_source_args - from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder - index_urls = [source.get("url") for source in self.sources] - - class PipCommand(Command): - name = "PipCommand" - - dependency_links = [] - packages = self.get_installed_packages() - # This code is borrowed from pip's current implementation - for dist in packages: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) - - pip_command = PipCommand() - index_opts = cmdoptions.make_option_group( - index_group, pip_command.parser - ) - cmd_opts = pip_command.cmd_opts - pip_command.parser.insert_option_group(0, index_opts) - pip_command.parser.insert_option_group(0, cmd_opts) - pip_args = prepare_pip_source_args(self.sources, []) - pip_options, _ = pip_command.parser.parse_args(pip_args) - pip_options.cache_dir = PIPENV_CACHE_DIR - pip_options.pre = self.settings.get("pre", False) - with pip_command._build_session(pip_options) as session: - finder = PackageFinder( - find_links=pip_options.find_links, - index_urls=index_urls, allow_all_prereleases=pip_options.pre, - trusted_hosts=pip_options.trusted_hosts, - process_dependency_links=pip_options.process_dependency_links, - session=session + @property + def environment(self): + if not self._environment: + prefix = self.get_location_for_virtualenv() + is_venv = prefix == sys.prefix + sources = self.sources.copy() if self.sources else [DEFAULT_SOURCE,] + self._environment = Environment( + prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile ) - finder.add_dependency_links(dependency_links) - - for dist in packages: - typ = 'unknown' - all_candidates = finder.find_all_candidates(dist.key) - if not pip_options.pre: - # Remove prereleases - all_candidates = [ - candidate for candidate in all_candidates - if not candidate.version.is_prerelease - ] - - if not all_candidates: - continue - best_candidate = max(all_candidates, key=finder._candidate_sort_key) - remote_version = best_candidate.version - if best_candidate.location.is_wheel: - typ = 'wheel' - else: - typ = 'sdist' - # This is dirty but makes the rest of the code much cleaner - dist.latest_version = remote_version - dist.latest_filetype = typ - yield dist + self._environment.add_dist("pipenv") + return self._environment def get_outdated_packages(self): - return [ - pkg for pkg in self.get_package_info() - if pkg.latest_version._version > pkg.parsed_version._version - ] + return self.environment.get_outdated_packages() @classmethod def _sanitize(cls, name): @@ -588,7 +532,6 @@ class Project(object): :return: A new toml hierarchical document """ - def gen_table(inline=False): if inline: return tomlkit.inline_table() @@ -1159,49 +1102,6 @@ class Project(object): # Return whether or not values have been changed. return changed_values - @property - def py_version(self): - py_path = self.which("python") - version = python_version(py_path) - return version - - @property - def _pyversion(self): - include_dir = vistir.compat.Path(self.virtualenv_location) / "include" - python_path = next((x for x in include_dir.iterdir() if x.name.startswith("python")), None) - if python_path: - py_version = python_path.name.replace("python", "") - py_version_short, abiflags = py_version[:3], py_version[3:] - return {"py_version_short": py_version_short, "abiflags": abiflags} - return {} - - @property - def env_paths(self): - location = self.virtualenv_location if self.virtualenv_location else sys.prefix - prefix = vistir.compat.Path(location) - import importlib - py_version = tuple([int(v) for v in self.py_version.split(".")]) - py_version_short = ".".join([str(v) for v in py_version[:2]]) - running_version = ".".join([str(v) for v in sys.version_info[:2]]) - try: - _virtualenv = importlib.import_module("virtualenv") - except (ImportError, AttributeError): - with vistir.contextmanagers.temp_path(): - sys.path = vistir.misc.load_path(self.which("python")) - six.moves.reload_module(importlib) - _virtualenv = importlib.import_module("virtualenv") - with sys_version(py_version): - home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix()) - paths = { - "lib": lib.replace(running_version, py_version_short), - "include": inc.replace(running_version, py_version_short), - "scripts": bin_, - "purelib": lib.replace(running_version, py_version_short), - "prefix": home, - "base": home - } - return paths - @cached_property def finders(self): from .vendor.pythonfinder import Finder From 642b6f94b55924ba5e3af199d6fd679e369a9ee6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 17:41:16 -0500 Subject: [PATCH 143/218] Update vistir and requirementslib Signed-off-by: Dan Ryan --- .../requirementslib/models/requirements.py | 25 ++++++---- .../requirementslib/models/setup_info.py | 2 +- pipenv/vendor/vistir/compat.py | 50 +++++++++++++++++-- pipenv/vendor/vistir/misc.py | 7 ++- pipenv/vendor/vistir/path.py | 31 +++++++----- pipenv/vendor/vistir/spin.py | 2 +- 6 files changed, 86 insertions(+), 31 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 51411429..aafb059b 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1050,9 +1050,9 @@ class Requirement(object): @classmethod def from_line(cls, line): - from pip_shims import InstallRequirement + import pip_shims.shims - if isinstance(line, InstallRequirement): + if isinstance(line, pip_shims.shims.InstallRequirement): line = format_requirement(line) hashes = None if "--hash=" in line: @@ -1130,13 +1130,20 @@ class Requirement(object): if hashes: args["hashes"] = hashes cls_inst = cls(**args) - if not cls_inst.is_named and (not cls_inst.editable or cls_inst.req._has_hashed_name): - old_name = cls_inst.req.req.name or cls_inst.req.name - info_dict = cls_inst.run_requires() - calced_name = info_dict.get("name", old_name) - if old_name != calced_name: - cls_inst.req.req.line.replace(old_name, calced_name) - cls_inst.name = cls_inst.req.name = calced_name + if not cls_inst.is_named and not cls_inst.editable and not name: + if cls_inst.is_vcs: + ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) + info = SetupInfo.from_ireq(ireq) + if info is not None: + info_dict = info.as_dict() + cls_inst.req.setup_info = info + else: + info_dict = {} + else: + info_dict = cls_inst.run_requires() + found_name = info_dict.get("name", old_name) + if old_name != found_name: + cls_inst.req.req.line.replace(old_name, found_name) return cls_inst @classmethod diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 319dd6bd..f2a1ee79 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -103,7 +103,7 @@ def iter_egginfos(path, pkg_name=None): def find_egginfo(target, pkg_name=None): egg_dirs = (egg_dir for egg_dir in iter_egginfos(target, pkg_name=pkg_name)) if pkg_name: - return next(iter(egg_dirs), None) + yield next(iter(egg_dirs), None) else: for egg_dir in egg_dirs: yield egg_dir diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index d6e8578a..83226481 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -20,6 +20,8 @@ __all__ = [ "FileNotFoundError", "ResourceWarning", "FileNotFoundError", + "PermissionError", + "IsADirectoryError", "fs_str", "lru_cache", "TemporaryDirectory", @@ -69,8 +71,17 @@ if six.PY2: self.errno = errno.ENOENT super(FileNotFoundError, self).__init__(*args, **kwargs) + class PermissionError(OSError): + def __init__(self, *args, **kwargs): + self.errno = errno.EACCES + super(PermissionError, self).__init__(*args, **kwargs) + + class IsADirectoryError(OSError): + """The command does not work on directories""" + pass + else: - from builtins import ResourceWarning, FileNotFoundError + from builtins import ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError if not sys.warnoptions: @@ -111,9 +122,39 @@ class TemporaryDirectory(object): ) @classmethod - def _cleanup(cls, name, warn_message): + def _rmtree(cls, name): from .path import rmtree - rmtree(name) + + def onerror(func, path, exc_info): + if issubclass(exc_info[0], (PermissionError, OSError)): + try: + try: + if path != name: + os.chflags(os.path.dirname(path), 0) + os.chflags(path, 0) + except AttributeError: + pass + if path != name: + os.chmod(os.path.dirname(path), 0o70) + os.chmod(path, 0o700) + + try: + os.unlink(path) + # PermissionError is raised on FreeBSD for directories + except (IsADirectoryError, PermissionError, OSError): + cls._rmtree(path) + except FileNotFoundError: + pass + elif issubclass(exc_info[0], FileNotFoundError): + pass + else: + raise + + rmtree(name, onerror=onerror) + + @classmethod + def _cleanup(cls, name, warn_message): + cls._rmtree(name) warnings.warn(warn_message, ResourceWarning) def __repr__(self): @@ -126,9 +167,8 @@ class TemporaryDirectory(object): self.cleanup() def cleanup(self): - from .path import rmtree if self._finalizer.detach(): - rmtree(self.name) + self._rmtree(self.name) def fs_str(string): diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 7342bc97..a9a127d8 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -35,7 +35,9 @@ __all__ = [ "locale_encoding", "chunked", "take", - "divide" + "divide", + "getpreferredencoding", + "decode_for_output", ] @@ -492,7 +494,8 @@ except Exception: def getpreferredencoding(): - import locale + """Determine the proper output encoding for terminal rendering""" + # Borrowed from Invoke # (see https://github.com/pyinvoke/invoke/blob/93af29d/invoke/runners.py#L881) _encoding = locale.getpreferredencoding(False) diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index b1236884..23ae0252 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -183,10 +183,9 @@ def mkdir_p(newdir, mode=0o777): :raises: OSError if a file is encountered along the way """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ - from .misc import to_text - from .compat import to_native_string + from .misc import to_bytes, to_text - newdir = to_native_string(newdir) + newdir = to_bytes(newdir, "utf-8") if os.path.exists(newdir): if not os.path.isdir(newdir): raise OSError( @@ -195,9 +194,9 @@ def mkdir_p(newdir, mode=0o777): ) ) else: - head, tail = os.path.split(newdir) + head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) # Make sure the tail doesn't point to the asame place as the head - curdir = to_native_string(".") + curdir = to_bytes(".", encoding="utf-8") tail_and_head_match = ( os.path.relpath(tail, start=os.path.basename(head)) == curdir ) @@ -205,8 +204,9 @@ def mkdir_p(newdir, mode=0o777): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): raise OSError( - "A file with the same name as the desired dir, '{0}', " - "already exists.".format(to_text(newdir, encoding="utf-8")) + "A file with the same name as the desired dir, '{0}', already exists.".format( + to_text(newdir, encoding="utf-8") + ) ) os.makedirs(os.path.join(head, tail), mode) @@ -277,13 +277,13 @@ def set_write_bit(fn): if not os.path.isdir(fn): return for root, dirs, files in os.walk(fn, topdown=False): - for dir_ in [os.path.join(root,d) for d in dirs]: + for dir_ in [os.path.join(root, d) for d in dirs]: set_write_bit(dir_) for file_ in [os.path.join(root, f) for f in files]: set_write_bit(file_) -def rmtree(directory, ignore_errors=False): +def rmtree(directory, ignore_errors=False, onerror=None): """Stand-in for :func:`~shutil.rmtree` with additional error-handling. This version of `rmtree` handles read-only paths, especially in the case of index @@ -291,6 +291,7 @@ def rmtree(directory, ignore_errors=False): :param str directory: The target directory to remove :param bool ignore_errors: Whether to ignore errors, defaults to False + :param func onerror: An error handling function, defaults to :func:`handle_remove_readonly` .. note:: @@ -300,9 +301,11 @@ def rmtree(directory, ignore_errors=False): from .compat import to_native_string directory = to_native_string(directory) + if onerror is None: + onerror = handle_remove_readonly try: shutil.rmtree( - directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly + directory, ignore_errors=ignore_errors, onerror=onerror ) except (IOError, OSError, FileNotFoundError) as exc: # Ignore removal failures where the file doesn't exist @@ -325,7 +328,9 @@ def handle_remove_readonly(func, path, exc): :func:`set_write_bit` on the target path and try again. """ # Check for read-only attribute - from .compat import ResourceWarning, FileNotFoundError, to_native_string + from .compat import ( + ResourceWarning, FileNotFoundError, PermissionError, to_native_string + ) PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) default_warning_message = ( @@ -339,7 +344,7 @@ def handle_remove_readonly(func, path, exc): set_write_bit(path) try: func(path) - except (OSError, IOError, FileNotFoundError) as e: + except (OSError, IOError, FileNotFoundError, PermissionError) as e: if e.errno == errno.ENOENT: return elif e.errno in PERM_ERRORS: @@ -350,7 +355,7 @@ def handle_remove_readonly(func, path, exc): set_write_bit(path) try: func(path) - except (OSError, IOError, FileNotFoundError) as e: + except (OSError, IOError, FileNotFoundError, PermissionError) as e: if e.errno in PERM_ERRORS: warnings.warn(default_warning_message.format(path), ResourceWarning) pass diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index f0d9e77f..09ecbace 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -292,6 +292,6 @@ class VistirSpinner(base_obj): def create_spinner(*args, **kwargs): nospin = kwargs.pop("nospin", False) use_yaspin = kwargs.pop("use_yaspin", nospin) - if nospin: + if nospin or not use_yaspin: return DummySpinner(*args, **kwargs) return VistirSpinner(*args, **kwargs) From aedb41c65d1a9a1e369d567f034827b9da1c9a3f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 17:44:35 -0500 Subject: [PATCH 144/218] Fix stdout and stderr wrappers Signed-off-by: Dan Ryan --- pipenv/__init__.py | 10 ++++++---- pipenv/resolver.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/pipenv/__init__.py b/pipenv/__init__.py index ba4dd9c3..4d137e7f 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -28,10 +28,12 @@ if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): if sys.stdout.isatty() and sys.stderr.isatty(): import io import atexit - sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') - atexit.register(sys.stdout.close) - sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') - atexit.register(sys.stdout.close) + stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') + atexit.register(stdout_wrapper.close) + stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') + atexit.register(stderr_wrapper.close) + sys.stdout = stdout_wrapper + sys.stderr = stderr_wrapper os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 9ef46878..e87f3243 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -99,8 +99,13 @@ def main(): import io import six if six.PY3: - sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') - sys.stderr = io.TextIOWrapper(sys.stderr.buffer,encoding='utf8') + import atexit + stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') + atexit.register(stdout_wrapper.close) + stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') + atexit.register(stderr_wrapper.close) + sys.stdout = stdout_wrapper + sys.stderr = stderr_wrapper else: from pipenv._compat import force_encoding force_encoding() @@ -111,7 +116,7 @@ def main(): # sys.argv = remaining parsed = handle_parsed_args(parsed) _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, - parsed.requirements_dir, parsed.packages) + parsed.requirements_dir, parsed.packages) if __name__ == "__main__": From 45100b8a46f10955e9cf89a6932b695219048695 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 17:44:35 -0500 Subject: [PATCH 145/218] Fix stdout and stderr wrappers Signed-off-by: Dan Ryan --- news/3196.vendor.rst | 1 + pipenv/__init__.py | 10 ++++++---- pipenv/resolver.py | 11 ++++++++--- 3 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 news/3196.vendor.rst diff --git a/news/3196.vendor.rst b/news/3196.vendor.rst new file mode 100644 index 00000000..19351e2e --- /dev/null +++ b/news/3196.vendor.rst @@ -0,0 +1 @@ +Updated ``requirementslib`` to aid in resolution of local and remote archives. diff --git a/pipenv/__init__.py b/pipenv/__init__.py index ba4dd9c3..4d137e7f 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -28,10 +28,12 @@ if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): if sys.stdout.isatty() and sys.stderr.isatty(): import io import atexit - sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') - atexit.register(sys.stdout.close) - sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') - atexit.register(sys.stdout.close) + stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') + atexit.register(stdout_wrapper.close) + stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') + atexit.register(stderr_wrapper.close) + sys.stdout = stdout_wrapper + sys.stderr = stderr_wrapper os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 9ef46878..e87f3243 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -99,8 +99,13 @@ def main(): import io import six if six.PY3: - sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') - sys.stderr = io.TextIOWrapper(sys.stderr.buffer,encoding='utf8') + import atexit + stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') + atexit.register(stdout_wrapper.close) + stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') + atexit.register(stderr_wrapper.close) + sys.stdout = stdout_wrapper + sys.stderr = stderr_wrapper else: from pipenv._compat import force_encoding force_encoding() @@ -111,7 +116,7 @@ def main(): # sys.argv = remaining parsed = handle_parsed_args(parsed) _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, - parsed.requirements_dir, parsed.packages) + parsed.requirements_dir, parsed.packages) if __name__ == "__main__": From 118c9d3fe69239150f54161b10d444e76f43f140 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 17:48:04 -0500 Subject: [PATCH 146/218] Add dramatically improved queued installation - Use queues and parallelized installation - Better UI/UX -- progress bar moves gradually as items are moved on and off the queue - Queue is handled by item instead of in massive batches - TODO: Call out when task is done from the install function? Signed-off-by: Dan Ryan --- ...90-3e70-40ba-8242-1e6ed18fc2fe.feature.rst | 1 + pipenv/core.py | 241 +++++++++--------- 2 files changed, 124 insertions(+), 118 deletions(-) create mode 100644 news/d65e7c90-3e70-40ba-8242-1e6ed18fc2fe.feature.rst diff --git a/news/d65e7c90-3e70-40ba-8242-1e6ed18fc2fe.feature.rst b/news/d65e7c90-3e70-40ba-8242-1e6ed18fc2fe.feature.rst new file mode 100644 index 00000000..f868fd95 --- /dev/null +++ b/news/d65e7c90-3e70-40ba-8242-1e6ed18fc2fe.feature.rst @@ -0,0 +1 @@ +Improved asynchronous installation and error handling via queued subprocess paralleization. diff --git a/pipenv/core.py b/pipenv/core.py index 021b0f3c..ef5439dd 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -654,6 +654,100 @@ def do_where(virtualenv=False, bare=True): click.echo(location) +def _cleanup_procs(procs, concurrent, failed_deps_queue, retry=True): + while not procs.empty(): + c = procs.get() + # if concurrent: + c.block() + failed = False + if c.return_code != 0: + failed = True + if "Ignoring" in c.out: + click.echo(crayons.yellow(c.out.strip())) + elif environments.is_verbose(): + click.echo(crayons.blue(c.out.strip() or c.err.strip())) + # The Installation failed… + if failed: + if not retry: + # The Installation failed… + # We echo both c.out and c.err because pip returns error details on out. + err = c.err.strip().splitlines() if c.err else [] + out = c.out.strip().splitlines() if c.out else [] + err_lines = [line for line in [out, err]] + # Return the subprocess' return code. + raise exceptions.InstallError(c.dep.name, extra=err_lines) + # Save the Failed Dependency for later. + dep = c.dep.copy() + failed_deps_queue.put(dep) + # Alert the user. + click.echo( + "{0} {1}! Will try again.".format( + crayons.red("An error occurred while installing"), + crayons.green(dep.as_line()), + ), err=True + ) + + +def batch_install(deps_list, procs, failed_deps_queue, + requirements_dir, no_deps=False, ignore_hashes=False, + allow_global=False, blocking=False, pypi_mirror=None, + nprocs=PIPENV_MAX_SUBPROCESS, retry=True): + + failed = (not retry) + if not failed: + label = INSTALL_LABEL if os.name != "nt" else "" + else: + label = INSTALL_LABEL2 + + deps_list_bar = progress.bar( + deps_list, width=32, + label=label + ) + indexes = [] + trusted_hosts = [] + # Install these because + for dep in deps_list_bar: + index = None + if dep.index: + index = project.find_source(dep.index) + indexes.append(index) + if not index.get("verify_ssl", False): + trusted_hosts.append(urllib3_util.parse_url(index.get("url")).host) + # Install the module. + is_artifact = False + if dep.is_file_or_url and (dep.is_direct_url or any( + dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] + )): + is_artifact = True + + extra_indexes = [] + if not index and indexes: + index = next(iter(indexes)) + if len(indexes) > 1: + extra_indexes = indexes[1:] + + with vistir.contextmanagers.temp_environ(): + os.environ["PIP_USER"] = vistir.compat.fs_str("0") + c = pip_install( + dep, + ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), + allow_global=allow_global, + no_deps=False if is_artifact else no_deps, + block=any([dep.is_vcs, blocking]), + index=index, + requirements_dir=requirements_dir, + pypi_mirror=pypi_mirror, + trusted_hosts=trusted_hosts, + extra_indexes=extra_indexes + ) + if procs.qsize() < nprocs: + c.dep = dep + procs.put(c) + + if procs.full() or procs.qsize() == len(deps_list): + _cleanup_procs(procs, not blocking, failed_deps_queue, retry=retry) + + def do_install_dependencies( dev=False, only=False, @@ -670,33 +764,8 @@ def do_install_dependencies( If requirements is True, simply spits out a requirements format to stdout. """ + from six.moves import queue - - def cleanup_procs(procs, concurrent): - while not procs.empty(): - c = procs.get() - # if concurrent: - c.block() - failed = False - if c.return_code != 0: - failed = True - if "Ignoring" in c.out: - click.echo(crayons.yellow(c.out.strip())) - elif environments.is_verbose(): - click.echo(crayons.blue(c.out.strip() or c.err.strip())) - # The Installation failed… - if failed: - # Save the Failed Dependency for later. - dep = c.dep.copy() - failed_deps_list.append(dep) - # Alert the user. - click.echo( - "{0} {1}! Will try again.".format( - crayons.red("An error occurred while installing"), - crayons.green(dep.as_line()), - ), err=True - ) - if requirements: bare = True blocking = not concurrent @@ -720,7 +789,6 @@ def do_install_dependencies( ) # Allow pip to resolve dependencies when in skip-lock mode. no_deps = not skip_lock - failed_deps_list = [] deps_list = list(lockfile.get_requirements(dev=dev, only=True)) if requirements: index_args = prepare_pip_source_args(project.sources) @@ -736,106 +804,43 @@ def do_install_dependencies( sys.exit(0) procs = queue.Queue(maxsize=PIPENV_MAX_SUBPROCESS) - trusted_hosts = [] + failed_deps_queue = queue.Queue() - deps_list_bar = progress.bar( - deps_list, width=32, - label=INSTALL_LABEL if os.name != "nt" else "", + install_kwargs = { + "no_deps": no_deps, "ignore_hashes": ignore_hashes, "allow_global": allow_global, + "blocking": blocking, "pypi_mirror": pypi_mirror + } + if concurrent: + install_kwargs["nprocs"] = PIPENV_MAX_SUBPROCESS + else: + install_kwargs["nprocs"] = 1 + + batch_install( + deps_list, procs, failed_deps_queue, requirements_dir, **install_kwargs ) - indexes = [] - for dep in deps_list_bar: - index = None - if dep.index: - index = project.find_source(dep.index) - indexes.append(index) - if not index.get("verify_ssl", False): - trusted_hosts.append(urllib3_util.parse_url(index.get("url")).host) - # Install the module. - is_artifact = False - if dep.is_file_or_url and any( - dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] - ): - is_artifact = True - extra_indexes = [] - if not index and indexes: - index = next(iter(indexes)) - if len(indexes) > 1: - extra_indexes = indexes[1:] - with vistir.contextmanagers.temp_environ(): - os.environ["PIP_USER"] = vistir.compat.fs_str("0") - c = pip_install( - dep, - ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), - allow_global=allow_global, - no_deps=False if is_artifact else no_deps, - block=any([dep.editable, blocking]), - index=index, - requirements_dir=requirements_dir, - pypi_mirror=pypi_mirror, - trusted_hosts=trusted_hosts, - extra_indexes=extra_indexes - ) - if procs.qsize() < PIPENV_MAX_SUBPROCESS: - c.dep = dep - procs.put(c) - - if procs.full() or procs.qsize() == len(deps_list): - cleanup_procs(procs, concurrent) if not procs.empty(): - cleanup_procs(procs, concurrent) + _cleanup_procs(procs, concurrent, failed_deps_queue) # Iterate over the hopefully-poorly-packaged dependencies… - if failed_deps_list: + if not failed_deps_queue.empty(): click.echo( crayons.normal(fix_utf8("Installing initially failed dependencies…"), bold=True) ) - for dep in progress.bar(failed_deps_list, label=INSTALL_LABEL2): - # Use a specific index, if specified. - # Install the module. - is_artifact = False - index = None - if dep.index: - index = project.find_source(dep.index) - if dep.is_file_or_url and any( - dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] - ): - is_artifact = True - extra_indexes = [] - if not index and indexes: - index = next(iter(indexes)) - if len(indexes) > 1: - extra_indexes = indexes[1:] - with vistir.contextmanagers.temp_environ(): - os.environ["PIP_USER"] = vistir.compat.fs_str("0") - c = pip_install( - dep, - ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), - allow_global=allow_global, - no_deps=True if is_artifact else no_deps, - index=index, - requirements_dir=requirements_dir, - pypi_mirror=pypi_mirror, - trusted_hosts=trusted_hosts, - extra_indexes=extra_indexes, - block=True - ) - # The Installation failed… - if c.return_code != 0: - # We echo both c.out and c.err because pip returns error details on out. - click.echo(crayons.blue(format_pip_output(c.out))) - click.echo(crayons.blue(format_pip_error(c.err)), err=True) - # Return the subprocess' return code. - sys.exit(c.return_code) - else: - if environments.is_verbose(): - click.echo( - "{0} {1}{2}".format( - crayons.green("Success installing"), - crayons.green(dep.as_line(include_hashes=False)), - crayons.green("!"), - ), - ) + retry_list = [] + while not failed_deps_queue.empty(): + failed_dep = failed_deps_queue.get() + retry_list.append(failed_dep) + install_kwargs.update({ + "nprocs": 1, + "retry": False, + "blocking": True, + }) + batch_install( + retry_list, procs, failed_deps_queue, requirements_dir, **install_kwargs + ) + if not procs.empty(): + _cleanup_procs(procs, False, failed_deps_queue, retry=False) def convert_three_to_python(three, python): From 8643a733c2af08bab269aabe99cc6f80c3d42e7f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 18:05:25 -0500 Subject: [PATCH 147/218] Fix configparser import Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/setup_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index f2a1ee79..481f0494 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -1,5 +1,4 @@ # -*- coding=utf-8 -*- -import configparser import contextlib import os @@ -14,6 +13,7 @@ except ImportError: import distutils from appdirs import user_cache_dir +from six.moves import configparser from six.moves.urllib.parse import unquote from vistir.compat import Path from vistir.contextmanagers import cd From 0caf7a013280d173354260e62ab17f3843ecd85d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 18:05:25 -0500 Subject: [PATCH 148/218] Fix configparser import Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/setup_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index f2a1ee79..481f0494 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -1,5 +1,4 @@ # -*- coding=utf-8 -*- -import configparser import contextlib import os @@ -14,6 +13,7 @@ except ImportError: import distutils from appdirs import user_cache_dir +from six.moves import configparser from six.moves.urllib.parse import unquote from vistir.compat import Path from vistir.contextmanagers import cd From 650cc32fe676619a3478b7712e48b511a6f2ac4b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 18:49:44 -0500 Subject: [PATCH 149/218] Fix resource errors Signed-off-by: Dan Ryan --- pipenv/environment.py | 2 +- pipenv/vendor/requirementslib/__init__.py | 5 ++--- pipenv/vendor/requirementslib/models/cache.py | 2 ++ pipenv/vendor/requirementslib/models/dependencies.py | 3 +++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index db0e22aa..5e867374 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -25,7 +25,7 @@ class Environment(object): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET - self.is_venv = not os.path.samefile(os.path.abspath(prefix), sys.prefix) + self.is_venv = not os.samefile(os.path.abspath(prefix), sys.prefix) if not sources: sources = [] self.sources = sources diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 881e9ac9..05fd1943 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -3,12 +3,11 @@ __version__ = '1.2.6' import logging import warnings - -warnings.filterwarnings("ignore", category=ResourceWarning) - +from vistir.compat import ResourceWarning logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) +warnings.filterwarnings("ignore", category=ResourceWarning) from .models.requirements import Requirement from .models.lockfile import Lockfile diff --git a/pipenv/vendor/requirementslib/models/cache.py b/pipenv/vendor/requirementslib/models/cache.py index 0b8c47b1..f1639ea2 100644 --- a/pipenv/vendor/requirementslib/models/cache.py +++ b/pipenv/vendor/requirementslib/models/cache.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals +import atexit import copy import hashlib import json @@ -197,6 +198,7 @@ class HashCache(SafeFileCache): if not session: import requests session = requests.session() + atexit.register(session.close) cache_dir = kwargs.pop('cache_dir', CACHE_DIR) self.session = session kwargs.setdefault('directory', os.path.join(cache_dir, 'hash-cache')) diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index 48e84d0f..f87fd585 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -1,5 +1,6 @@ # -*- coding=utf-8 -*- +import atexit import contextlib import copy import functools @@ -361,6 +362,7 @@ def get_dependencies_from_json(ireq): return session = requests.session() + atexit.register(session.close) version = str(ireq.req.specifier).lstrip("=") def gen(ireq): @@ -575,6 +577,7 @@ def get_finder(sources=None, pip_command=None, pip_options=None): if not pip_options: pip_options = get_pip_options(sources=sources, pip_command=pip_command) session = pip_command._build_session(pip_options) + atexit.register(session.close) finder = pip_shims.shims.PackageFinder( find_links=[], index_urls=[s.get("url") for s in sources], From fe9d996f89aa98e047d795ae6cea7957986ccc26 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 18:49:44 -0500 Subject: [PATCH 150/218] Fix resource errors Signed-off-by: Dan Ryan --- pipenv/environment.py | 2 +- pipenv/vendor/requirementslib/__init__.py | 5 ++--- pipenv/vendor/requirementslib/models/cache.py | 2 ++ pipenv/vendor/requirementslib/models/dependencies.py | 3 +++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index db0e22aa..5e867374 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -25,7 +25,7 @@ class Environment(object): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET - self.is_venv = not os.path.samefile(os.path.abspath(prefix), sys.prefix) + self.is_venv = not os.samefile(os.path.abspath(prefix), sys.prefix) if not sources: sources = [] self.sources = sources diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 881e9ac9..05fd1943 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -3,12 +3,11 @@ __version__ = '1.2.6' import logging import warnings - -warnings.filterwarnings("ignore", category=ResourceWarning) - +from vistir.compat import ResourceWarning logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) +warnings.filterwarnings("ignore", category=ResourceWarning) from .models.requirements import Requirement from .models.lockfile import Lockfile diff --git a/pipenv/vendor/requirementslib/models/cache.py b/pipenv/vendor/requirementslib/models/cache.py index 0b8c47b1..f1639ea2 100644 --- a/pipenv/vendor/requirementslib/models/cache.py +++ b/pipenv/vendor/requirementslib/models/cache.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals +import atexit import copy import hashlib import json @@ -197,6 +198,7 @@ class HashCache(SafeFileCache): if not session: import requests session = requests.session() + atexit.register(session.close) cache_dir = kwargs.pop('cache_dir', CACHE_DIR) self.session = session kwargs.setdefault('directory', os.path.join(cache_dir, 'hash-cache')) diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index 48e84d0f..f87fd585 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -1,5 +1,6 @@ # -*- coding=utf-8 -*- +import atexit import contextlib import copy import functools @@ -361,6 +362,7 @@ def get_dependencies_from_json(ireq): return session = requests.session() + atexit.register(session.close) version = str(ireq.req.specifier).lstrip("=") def gen(ireq): @@ -575,6 +577,7 @@ def get_finder(sources=None, pip_command=None, pip_options=None): if not pip_options: pip_options = get_pip_options(sources=sources, pip_command=pip_command) session = pip_command._build_session(pip_options) + atexit.register(session.close) finder = pip_shims.shims.PackageFinder( find_links=[], index_urls=[s.get("url") for s in sources], From e5be2ac50ac1e26f65bad5844bb05db74c53ce36 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 19:03:55 -0500 Subject: [PATCH 151/218] Fix python 2.7 installations Signed-off-by: Dan Ryan --- pipenv/core.py | 20 ++++++++++++-------- pipenv/environment.py | 2 +- pipenv/vendor/requirementslib/exceptions.py | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 40fc87c4..99184c3d 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1852,7 +1852,7 @@ def do_install( # Install all dependencies, if none was provided. # This basically ensures that we have a pipfile and lockfile, then it locks and # installs from the lockfile - if packages is False and editable_packages is False: + if not packages and not editable_packages: # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) @@ -1877,13 +1877,17 @@ def do_install( # make a tuple of (display_name, entry) pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages] if not system and not project.virtualenv_exists: - with create_spinner("Creating virtualenv...") as sp: - try: - do_create_virtualenv(pypi_mirror=pypi_mirror) - except KeyboardInterrupt: - cleanup_virtualenv(bare=(not environments.is_verbose())) - sys.exit(1) - sp.write_err("Ok...") + do_init( + dev=dev, + system=system, + allow_global=system, + concurrent=concurrent, + keep_outdated=keep_outdated, + requirements_dir=requirements_directory, + deploy=deploy, + pypi_mirror=pypi_mirror, + skip_lock=skip_lock, + ) for pkg_line in pkg_list: click.echo( crayons.normal( diff --git a/pipenv/environment.py b/pipenv/environment.py index 5e867374..db0e22aa 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -25,7 +25,7 @@ class Environment(object): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET - self.is_venv = not os.samefile(os.path.abspath(prefix), sys.prefix) + self.is_venv = not os.path.samefile(os.path.abspath(prefix), sys.prefix) if not sources: sources = [] self.sources = sources diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/vendor/requirementslib/exceptions.py index 1a73f98e..23bc5e50 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/vendor/requirementslib/exceptions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import errno import os import six From 382be38bbabb3b65425148548bd51fddc1fb28e1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 19:03:55 -0500 Subject: [PATCH 152/218] Fix python 2.7 installations Signed-off-by: Dan Ryan --- pipenv/core.py | 20 ++++++++++++-------- pipenv/environment.py | 2 +- pipenv/vendor/requirementslib/exceptions.py | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index c5891a57..3dd126bf 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1847,7 +1847,7 @@ def do_install( # Install all dependencies, if none was provided. # This basically ensures that we have a pipfile and lockfile, then it locks and # installs from the lockfile - if packages is False and editable_packages is False: + if not packages and not editable_packages: # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) @@ -1872,13 +1872,17 @@ def do_install( # make a tuple of (display_name, entry) pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages] if not system and not project.virtualenv_exists: - with create_spinner("Creating virtualenv...") as sp: - try: - do_create_virtualenv(pypi_mirror=pypi_mirror) - except KeyboardInterrupt: - cleanup_virtualenv(bare=(not environments.is_verbose())) - sys.exit(1) - sp.write_err("Ok...") + do_init( + dev=dev, + system=system, + allow_global=system, + concurrent=concurrent, + keep_outdated=keep_outdated, + requirements_dir=requirements_directory, + deploy=deploy, + pypi_mirror=pypi_mirror, + skip_lock=skip_lock, + ) for pkg_line in pkg_list: click.echo( crayons.normal( diff --git a/pipenv/environment.py b/pipenv/environment.py index 5e867374..db0e22aa 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -25,7 +25,7 @@ class Environment(object): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET - self.is_venv = not os.samefile(os.path.abspath(prefix), sys.prefix) + self.is_venv = not os.path.samefile(os.path.abspath(prefix), sys.prefix) if not sources: sources = [] self.sources = sources diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/vendor/requirementslib/exceptions.py index 1a73f98e..23bc5e50 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/vendor/requirementslib/exceptions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import errno import os import six From 489e534c9ae0d912390a2681b0887839d492bbf2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 23:38:09 -0500 Subject: [PATCH 153/218] Fix various bugs with python 2.7 and vendored deps Signed-off-by: Dan Ryan --- pipenv/project.py | 2 +- .../requirementslib/models/setup_info.py | 30 ++++++++++++++++--- pipenv/vendor/vistir/contextmanagers.py | 8 +++-- pipenv/vendor/vistir/spin.py | 2 +- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index fbc18f19..0eafff8e 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -350,7 +350,7 @@ class Project(object): if not self._environment: prefix = self.get_location_for_virtualenv() is_venv = prefix == sys.prefix - sources = self.sources.copy() if self.sources else [DEFAULT_SOURCE,] + sources = self.sources if self.sources else [DEFAULT_SOURCE,] self._environment = Environment( prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile ) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 481f0494..247d63f2 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- import contextlib import os +import sys import attr import packaging.version @@ -29,6 +30,11 @@ except ImportError: CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) +# The following are necessary for people who like to use "if __name__" conditionals +# in their setup.py scripts +_setup_stop_after = None +_setup_distribution = None + @contextlib.contextmanager def _suppress_distutils_logs(): @@ -116,7 +122,7 @@ def get_metadata(path, pkg_name=None): if egg_dir is not None: import pkg_resources - egg_dir = os.path.abspath(egg_dir) + egg_dir = os.path.abspath(egg_dir.path) base_dir = os.path.dirname(egg_dir) path_metadata = pkg_resources.PathMetadata(base_dir, egg_dir) dist = next( @@ -216,10 +222,26 @@ class SetupInfo(object): if self.setup_py is not None and self.setup_py.exists(): with cd(self.setup_py.parent), _suppress_distutils_logs(): from setuptools.dist import distutils + save_argv = sys.argv.copy() + try: + # This is for you, Hynek + # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py + global _setup_distribution, _setup_stop_after + _setup_stop_after = "run" + script_name = self.setup_py.as_posix() + g = {"__file__": script_name, "__name__": "__main__"} + sys.argv[0] = script_name + sys.argv[1:] = ["egg_info", "--egg-base", self.base_dir] + with open(script_name, 'rb') as f: + exec(f.read(), g) + finally: + _setup_stop_after = None + sys.argv = save_argv + dist = _setup_distribution + if not dist: + self.get_egg_metadata() + return - dist = distutils.core.run_setup( - self.setup_py.as_posix(), ["egg_info", "--egg-base", self.base_dir] - ) name = dist.get_name() if name: self.name = name diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 59b97ca0..3f191120 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -118,10 +118,11 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): """ from .spin import create_spinner - has_yaspin = False + has_yaspin = None try: import yaspin except ImportError: + has_yaspin = False if not nospin: raise RuntimeError( "Failed to import spinner! Reinstall vistir with command:" @@ -132,6 +133,9 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): else: has_yaspin = True spinner_name = "" + use_yaspin = (has_yaspin is False) or (nospin is True) + if has_yaspin is None or has_yaspin is True and not nospin: + use_yaspin = True if not start_text and nospin is False: start_text = "Running..." with create_spinner( @@ -139,7 +143,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): text=start_text, handler_map=handler_map, nospin=nospin, - use_yaspin=has_yaspin + use_yaspin=use_yaspin ) as _spinner: yield _spinner diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index 09ecbace..e7311555 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -291,7 +291,7 @@ class VistirSpinner(base_obj): def create_spinner(*args, **kwargs): nospin = kwargs.pop("nospin", False) - use_yaspin = kwargs.pop("use_yaspin", nospin) + use_yaspin = kwargs.pop("use_yaspin", not nospin) if nospin or not use_yaspin: return DummySpinner(*args, **kwargs) return VistirSpinner(*args, **kwargs) From 398463245332633d80ba87dcf664a645b0ed233d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 23:38:09 -0500 Subject: [PATCH 154/218] Fix various bugs with python 2.7 and vendored deps Signed-off-by: Dan Ryan --- pipenv/project.py | 2 +- .../requirementslib/models/setup_info.py | 30 ++++++++++++++++--- pipenv/vendor/vistir/contextmanagers.py | 8 +++-- pipenv/vendor/vistir/spin.py | 2 +- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index fbc18f19..0eafff8e 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -350,7 +350,7 @@ class Project(object): if not self._environment: prefix = self.get_location_for_virtualenv() is_venv = prefix == sys.prefix - sources = self.sources.copy() if self.sources else [DEFAULT_SOURCE,] + sources = self.sources if self.sources else [DEFAULT_SOURCE,] self._environment = Environment( prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile ) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 481f0494..247d63f2 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- import contextlib import os +import sys import attr import packaging.version @@ -29,6 +30,11 @@ except ImportError: CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) +# The following are necessary for people who like to use "if __name__" conditionals +# in their setup.py scripts +_setup_stop_after = None +_setup_distribution = None + @contextlib.contextmanager def _suppress_distutils_logs(): @@ -116,7 +122,7 @@ def get_metadata(path, pkg_name=None): if egg_dir is not None: import pkg_resources - egg_dir = os.path.abspath(egg_dir) + egg_dir = os.path.abspath(egg_dir.path) base_dir = os.path.dirname(egg_dir) path_metadata = pkg_resources.PathMetadata(base_dir, egg_dir) dist = next( @@ -216,10 +222,26 @@ class SetupInfo(object): if self.setup_py is not None and self.setup_py.exists(): with cd(self.setup_py.parent), _suppress_distutils_logs(): from setuptools.dist import distutils + save_argv = sys.argv.copy() + try: + # This is for you, Hynek + # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py + global _setup_distribution, _setup_stop_after + _setup_stop_after = "run" + script_name = self.setup_py.as_posix() + g = {"__file__": script_name, "__name__": "__main__"} + sys.argv[0] = script_name + sys.argv[1:] = ["egg_info", "--egg-base", self.base_dir] + with open(script_name, 'rb') as f: + exec(f.read(), g) + finally: + _setup_stop_after = None + sys.argv = save_argv + dist = _setup_distribution + if not dist: + self.get_egg_metadata() + return - dist = distutils.core.run_setup( - self.setup_py.as_posix(), ["egg_info", "--egg-base", self.base_dir] - ) name = dist.get_name() if name: self.name = name diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 59b97ca0..3f191120 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -118,10 +118,11 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): """ from .spin import create_spinner - has_yaspin = False + has_yaspin = None try: import yaspin except ImportError: + has_yaspin = False if not nospin: raise RuntimeError( "Failed to import spinner! Reinstall vistir with command:" @@ -132,6 +133,9 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): else: has_yaspin = True spinner_name = "" + use_yaspin = (has_yaspin is False) or (nospin is True) + if has_yaspin is None or has_yaspin is True and not nospin: + use_yaspin = True if not start_text and nospin is False: start_text = "Running..." with create_spinner( @@ -139,7 +143,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): text=start_text, handler_map=handler_map, nospin=nospin, - use_yaspin=has_yaspin + use_yaspin=use_yaspin ) as _spinner: yield _spinner diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index 09ecbace..e7311555 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -291,7 +291,7 @@ class VistirSpinner(base_obj): def create_spinner(*args, **kwargs): nospin = kwargs.pop("nospin", False) - use_yaspin = kwargs.pop("use_yaspin", nospin) + use_yaspin = kwargs.pop("use_yaspin", not nospin) if nospin or not use_yaspin: return DummySpinner(*args, **kwargs) return VistirSpinner(*args, **kwargs) From 32a6dd38a56e40b408317dc922b0e9f703b9be71 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 23:58:52 -0500 Subject: [PATCH 155/218] Support python 2 parsing Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/setup_info.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 247d63f2..561ba156 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -221,19 +221,22 @@ class SetupInfo(object): def run_setup(self): if self.setup_py is not None and self.setup_py.exists(): with cd(self.setup_py.parent), _suppress_distutils_logs(): - from setuptools.dist import distutils - save_argv = sys.argv.copy() - try: + if sys.version_info < (3, 5): + save_argv = sys.argv[:] + else: + save_argv = sys.argv.copy() # This is for you, Hynek # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py + try: global _setup_distribution, _setup_stop_after _setup_stop_after = "run" script_name = self.setup_py.as_posix() g = {"__file__": script_name, "__name__": "__main__"} + l = {} sys.argv[0] = script_name sys.argv[1:] = ["egg_info", "--egg-base", self.base_dir] with open(script_name, 'rb') as f: - exec(f.read(), g) + exec(f.read(), g, l) finally: _setup_stop_after = None sys.argv = save_argv From 4009198340db2086dca831f445177ba9e1e26146 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 00:08:28 -0500 Subject: [PATCH 156/218] Fix environment site import Signed-off-by: Dan Ryan --- pipenv/environment.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index db0e22aa..c3c33dda 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -7,6 +7,7 @@ import os import sys import operator import pkg_resources +import site import six from distutils.sysconfig import get_python_lib @@ -239,7 +240,6 @@ class Environment(object): return pkg_resources.find_distributions(self.paths["PYTHONPATH"]) def find_egg(self, egg_dist): - import site site_packages = get_python_lib() search_filename = "{0}.egg-link".format(egg_dist.project_name) try: @@ -476,7 +476,6 @@ class Environment(object): os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix) sys.path = self.sys_path sys.prefix = self.sys_prefix - site = self.safe_import("site") site.addsitedir(self.base_paths["purelib"]) if include_extras: site.addsitedir(parent_path) From 32b1113fbb6bad61f138f1aa1c47d1a686f35baa Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 11 Nov 2018 23:58:52 -0500 Subject: [PATCH 157/218] Support python 2 parsing Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/setup_info.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 247d63f2..561ba156 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -221,19 +221,22 @@ class SetupInfo(object): def run_setup(self): if self.setup_py is not None and self.setup_py.exists(): with cd(self.setup_py.parent), _suppress_distutils_logs(): - from setuptools.dist import distutils - save_argv = sys.argv.copy() - try: + if sys.version_info < (3, 5): + save_argv = sys.argv[:] + else: + save_argv = sys.argv.copy() # This is for you, Hynek # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py + try: global _setup_distribution, _setup_stop_after _setup_stop_after = "run" script_name = self.setup_py.as_posix() g = {"__file__": script_name, "__name__": "__main__"} + l = {} sys.argv[0] = script_name sys.argv[1:] = ["egg_info", "--egg-base", self.base_dir] with open(script_name, 'rb') as f: - exec(f.read(), g) + exec(f.read(), g, l) finally: _setup_stop_after = None sys.argv = save_argv From 1216ae0c8a898ce81a3261762cf759c1d9fe5c4f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 00:08:28 -0500 Subject: [PATCH 158/218] Fix environment site import Signed-off-by: Dan Ryan --- pipenv/environment.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index db0e22aa..c3c33dda 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -7,6 +7,7 @@ import os import sys import operator import pkg_resources +import site import six from distutils.sysconfig import get_python_lib @@ -239,7 +240,6 @@ class Environment(object): return pkg_resources.find_distributions(self.paths["PYTHONPATH"]) def find_egg(self, egg_dist): - import site site_packages = get_python_lib() search_filename = "{0}.egg-link".format(egg_dist.project_name) try: @@ -476,7 +476,6 @@ class Environment(object): os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix) sys.path = self.sys_path sys.prefix = self.sys_prefix - site = self.safe_import("site") site.addsitedir(self.base_paths["purelib"]) if include_extras: site.addsitedir(parent_path) From 70fc92b08d213209a7f4363c616fe490cef9dc66 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 10:05:21 -0500 Subject: [PATCH 159/218] Fix import errors on setup parsing Signed-off-by: Dan Ryan --- .../requirementslib/models/setup_info.py | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 561ba156..6107a240 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -18,6 +18,7 @@ from six.moves import configparser from six.moves.urllib.parse import unquote from vistir.compat import Path from vistir.contextmanagers import cd +from vistir.misc import run from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p from .utils import init_requirement, get_pyproject @@ -220,9 +221,15 @@ class SetupInfo(object): def run_setup(self): if self.setup_py is not None and self.setup_py.exists(): - with cd(self.setup_py.parent), _suppress_distutils_logs(): + target_cwd = self.setup_py.parent.as_posix() + with cd(target_cwd), _suppress_distutils_logs(): + from setuptools.dist import distutils + script_name = self.setup_py.as_posix() + args = ["egg_info", "--egg-base", self.base_dir] + g = {"__file__": script_name, "__name__": "__main__"} + local_dict = {} if sys.version_info < (3, 5): - save_argv = sys.argv[:] + save_argv = sys.argv else: save_argv = sys.argv.copy() # This is for you, Hynek @@ -230,13 +237,18 @@ class SetupInfo(object): try: global _setup_distribution, _setup_stop_after _setup_stop_after = "run" - script_name = self.setup_py.as_posix() - g = {"__file__": script_name, "__name__": "__main__"} - l = {} sys.argv[0] = script_name - sys.argv[1:] = ["egg_info", "--egg-base", self.base_dir] + sys.argv[1:] = args with open(script_name, 'rb') as f: - exec(f.read(), g, l) + if sys.version_info < (3, 5): + exec(f.read(), g, local_dict) + else: + exec(f.read(), g) + # We couldn't import everything needed to run setup + except NameError: + python = os.environ.get('PIP_PYTHON_PATH', sys.executable) + out, _ = run([python, "setup.py"] + args, cwd=target_cwd, block=True, + combine_stderr=False, return_object=False, nospin=True) finally: _setup_stop_after = None sys.argv = save_argv From 7e139ad981c44c65c1f2e18d9caed64fc874496f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 10:05:21 -0500 Subject: [PATCH 160/218] Fix import errors on setup parsing Signed-off-by: Dan Ryan --- .../requirementslib/models/setup_info.py | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 561ba156..6107a240 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -18,6 +18,7 @@ from six.moves import configparser from six.moves.urllib.parse import unquote from vistir.compat import Path from vistir.contextmanagers import cd +from vistir.misc import run from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p from .utils import init_requirement, get_pyproject @@ -220,9 +221,15 @@ class SetupInfo(object): def run_setup(self): if self.setup_py is not None and self.setup_py.exists(): - with cd(self.setup_py.parent), _suppress_distutils_logs(): + target_cwd = self.setup_py.parent.as_posix() + with cd(target_cwd), _suppress_distutils_logs(): + from setuptools.dist import distutils + script_name = self.setup_py.as_posix() + args = ["egg_info", "--egg-base", self.base_dir] + g = {"__file__": script_name, "__name__": "__main__"} + local_dict = {} if sys.version_info < (3, 5): - save_argv = sys.argv[:] + save_argv = sys.argv else: save_argv = sys.argv.copy() # This is for you, Hynek @@ -230,13 +237,18 @@ class SetupInfo(object): try: global _setup_distribution, _setup_stop_after _setup_stop_after = "run" - script_name = self.setup_py.as_posix() - g = {"__file__": script_name, "__name__": "__main__"} - l = {} sys.argv[0] = script_name - sys.argv[1:] = ["egg_info", "--egg-base", self.base_dir] + sys.argv[1:] = args with open(script_name, 'rb') as f: - exec(f.read(), g, l) + if sys.version_info < (3, 5): + exec(f.read(), g, local_dict) + else: + exec(f.read(), g) + # We couldn't import everything needed to run setup + except NameError: + python = os.environ.get('PIP_PYTHON_PATH', sys.executable) + out, _ = run([python, "setup.py"] + args, cwd=target_cwd, block=True, + combine_stderr=False, return_object=False, nospin=True) finally: _setup_stop_after = None sys.argv = save_argv From 8502ac96ece321f75b2c10e641d8d2ffba9c9089 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 12:04:58 -0500 Subject: [PATCH 161/218] Revendor Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 1 - pipenv/vendor/pythonfinder/pythonfinder.py | 4 ++-- pipenv/vendor/pythonfinder/utils.py | 2 +- pipenv/vendor/requirementslib/models/lockfile.py | 4 ++-- .../vendor/requirementslib/models/requirements.py | 15 +++++---------- pipenv/vendor/requirementslib/models/utils.py | 1 - pipenv/vendor/vendor.txt | 6 +++--- 7 files changed, 13 insertions(+), 20 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 33b4ab58..3d01e7cf 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -448,7 +448,6 @@ class PathEntry(BasePath): if self.is_dir: return None if self.is_python: - from .python import PythonVersion try: py_version = PythonVersion.from_path(path=self, name=self.name) except InvalidPythonVersion: diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 854cc8e7..b3bad570 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -65,7 +65,7 @@ class Finder(object): def which(self, exe): return self.system_path.which(exe) - @lru_cache(maxsize=128) + @lru_cache(maxsize=1024) def find_python_version( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): @@ -113,7 +113,7 @@ class Finder(object): major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) - @lru_cache(maxsize=128) + @lru_cache(maxsize=1024) def find_all_python_versions( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index ca07b42f..42a63e54 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -90,7 +90,7 @@ def looks_like_python(name): return any(fnmatch(name, rule) for rule in MATCH_RULES) -@lru_cache(maxsize=128) +@lru_cache(maxsize=1024) def path_is_python(path): return path_is_executable(path) and looks_like_python(path.name) diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index 6f61f57e..9d19edaf 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -223,13 +223,13 @@ class Lockfile(object): try: projectfile = cls.load_projectfile(path, create=create) - except JSONDecodeError as e: + except JSONDecodeError: path = os.path.abspath(path) if not os.path.isdir(path): path = os.path.dirname(path) path = Path(os.path.join(path, "Pipfile.lock")) formatted_path = path.as_posix() - backup_path = "%.bak" % formatted_path + backup_path = "%s.bak" % formatted_path LockfileCorruptException.show(formatted_path, backup_path=backup_path) path.rename(backup_path) cls.load(formatted_path, create=True) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index aafb059b..d5330b48 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -14,7 +14,7 @@ import pip_shims from first import first from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement -from packaging.specifiers import Specifier, SpecifierSet +from packaging.specifiers import Specifier, SpecifierSet, LegacySpecifier, InvalidSpecifier from packaging.utils import canonicalize_name from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote @@ -325,9 +325,6 @@ class FileRequirement(object): if setup_name: name = setup_name self._has_hashed_name = False - version = setupinfo_dict.get("version") - if version and not self.version: - self.version = version build_requires = setupinfo_dict.get("build_requires") build_backend = setupinfo_dict.get("build_backend") if build_requires and not self.pyproject_requires: @@ -404,7 +401,6 @@ class FileRequirement(object): cls, path=None, uri=None, editable=False, extras=None, link=None, vcs_type=None, name=None, req=None, line=None, uri_scheme=None, setup_path=None, relpath=None ): - import pip_shims.shims if relpath and not path: path = relpath if not path and uri and link.scheme == "file": @@ -455,7 +451,6 @@ class FileRequirement(object): creation_kwargs["vcs_type"] = vcs_type _line = None if not name: - import pip_shims.shims _line = unquote(link.url_without_fragment) if link.url else uri if editable: ireq = pip_shims.shims.install_req_from_editable(_line) @@ -1050,8 +1045,6 @@ class Requirement(object): @classmethod def from_line(cls, line): - import pip_shims.shims - if isinstance(line, pip_shims.shims.InstallRequirement): line = format_requirement(line) hashes = None @@ -1200,7 +1193,6 @@ class Requirement(object): old_name = cls_inst.req.req.name or cls_inst.req.name if not cls_inst.is_named and not cls_inst.editable and not name: if cls_inst.is_vcs: - import pip_shims.shims ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) info = SetupInfo.from_ireq(ireq) if info is not None: @@ -1276,7 +1268,10 @@ class Requirement(object): return markers def get_specifier(self): - return Specifier(self.specifiers) + try: + return Specifier(self.specifiers) + except InvalidSpecifier: + return LegacySpecifier(self.specifiers) def get_version(self): return pip_shims.shims.parse_version(self.get_specifier().version) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 2b47ee9b..0fac2aa3 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -95,7 +95,6 @@ def build_vcs_link(vcs, uri, name=None, ref=None, subdirectory=None, extras=None if extras: extras = extras_to_string(extras) uri = "{0}{1}".format(uri, extras) - # if subdirectory: if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) return create_link(uri) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 45ff0384..c106a59c 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -21,20 +21,20 @@ pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.7 +pythonfinder==1.1.8 requests==2.20.0 chardet==3.0.4 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.2.5 +requirementslib==1.3.0 attrs==18.2.0 distlib==0.2.8 packaging==18.0 pyparsing==2.2.2 pytoml==0.1.19 plette==0.2.2 - tomlkit==0.4.6 + tomlkit==0.5.2 shellingham==1.2.7 six==1.11.0 semver==2.8.1 From 013e3d0ec22607769ac749544c1e5f20ade261c2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 12:04:58 -0500 Subject: [PATCH 162/218] Revendor Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 1 - pipenv/vendor/pythonfinder/pythonfinder.py | 4 ++-- pipenv/vendor/pythonfinder/utils.py | 2 +- pipenv/vendor/requirementslib/models/lockfile.py | 4 ++-- .../vendor/requirementslib/models/requirements.py | 15 +++++---------- pipenv/vendor/requirementslib/models/utils.py | 1 - pipenv/vendor/vendor.txt | 6 +++--- 7 files changed, 13 insertions(+), 20 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 33b4ab58..3d01e7cf 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -448,7 +448,6 @@ class PathEntry(BasePath): if self.is_dir: return None if self.is_python: - from .python import PythonVersion try: py_version = PythonVersion.from_path(path=self, name=self.name) except InvalidPythonVersion: diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 854cc8e7..b3bad570 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -65,7 +65,7 @@ class Finder(object): def which(self, exe): return self.system_path.which(exe) - @lru_cache(maxsize=128) + @lru_cache(maxsize=1024) def find_python_version( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): @@ -113,7 +113,7 @@ class Finder(object): major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) - @lru_cache(maxsize=128) + @lru_cache(maxsize=1024) def find_all_python_versions( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index ca07b42f..42a63e54 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -90,7 +90,7 @@ def looks_like_python(name): return any(fnmatch(name, rule) for rule in MATCH_RULES) -@lru_cache(maxsize=128) +@lru_cache(maxsize=1024) def path_is_python(path): return path_is_executable(path) and looks_like_python(path.name) diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index 6f61f57e..9d19edaf 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -223,13 +223,13 @@ class Lockfile(object): try: projectfile = cls.load_projectfile(path, create=create) - except JSONDecodeError as e: + except JSONDecodeError: path = os.path.abspath(path) if not os.path.isdir(path): path = os.path.dirname(path) path = Path(os.path.join(path, "Pipfile.lock")) formatted_path = path.as_posix() - backup_path = "%.bak" % formatted_path + backup_path = "%s.bak" % formatted_path LockfileCorruptException.show(formatted_path, backup_path=backup_path) path.rename(backup_path) cls.load(formatted_path, create=True) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index aafb059b..d5330b48 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -14,7 +14,7 @@ import pip_shims from first import first from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement -from packaging.specifiers import Specifier, SpecifierSet +from packaging.specifiers import Specifier, SpecifierSet, LegacySpecifier, InvalidSpecifier from packaging.utils import canonicalize_name from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote @@ -325,9 +325,6 @@ class FileRequirement(object): if setup_name: name = setup_name self._has_hashed_name = False - version = setupinfo_dict.get("version") - if version and not self.version: - self.version = version build_requires = setupinfo_dict.get("build_requires") build_backend = setupinfo_dict.get("build_backend") if build_requires and not self.pyproject_requires: @@ -404,7 +401,6 @@ class FileRequirement(object): cls, path=None, uri=None, editable=False, extras=None, link=None, vcs_type=None, name=None, req=None, line=None, uri_scheme=None, setup_path=None, relpath=None ): - import pip_shims.shims if relpath and not path: path = relpath if not path and uri and link.scheme == "file": @@ -455,7 +451,6 @@ class FileRequirement(object): creation_kwargs["vcs_type"] = vcs_type _line = None if not name: - import pip_shims.shims _line = unquote(link.url_without_fragment) if link.url else uri if editable: ireq = pip_shims.shims.install_req_from_editable(_line) @@ -1050,8 +1045,6 @@ class Requirement(object): @classmethod def from_line(cls, line): - import pip_shims.shims - if isinstance(line, pip_shims.shims.InstallRequirement): line = format_requirement(line) hashes = None @@ -1200,7 +1193,6 @@ class Requirement(object): old_name = cls_inst.req.req.name or cls_inst.req.name if not cls_inst.is_named and not cls_inst.editable and not name: if cls_inst.is_vcs: - import pip_shims.shims ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) info = SetupInfo.from_ireq(ireq) if info is not None: @@ -1276,7 +1268,10 @@ class Requirement(object): return markers def get_specifier(self): - return Specifier(self.specifiers) + try: + return Specifier(self.specifiers) + except InvalidSpecifier: + return LegacySpecifier(self.specifiers) def get_version(self): return pip_shims.shims.parse_version(self.get_specifier().version) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 2b47ee9b..0fac2aa3 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -95,7 +95,6 @@ def build_vcs_link(vcs, uri, name=None, ref=None, subdirectory=None, extras=None if extras: extras = extras_to_string(extras) uri = "{0}{1}".format(uri, extras) - # if subdirectory: if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) return create_link(uri) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 45ff0384..c106a59c 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -21,20 +21,20 @@ pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.7 +pythonfinder==1.1.8 requests==2.20.0 chardet==3.0.4 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.2.5 +requirementslib==1.3.0 attrs==18.2.0 distlib==0.2.8 packaging==18.0 pyparsing==2.2.2 pytoml==0.1.19 plette==0.2.2 - tomlkit==0.4.6 + tomlkit==0.5.2 shellingham==1.2.7 six==1.11.0 semver==2.8.1 From 96cbd58d84a04d319c422e3e2fcc96c184e48efb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 13:36:58 -0500 Subject: [PATCH 163/218] Fix prefix comparison for py2 Signed-off-by: Dan Ryan --- pipenv/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index c3c33dda..b96f2fbe 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -26,7 +26,7 @@ class Environment(object): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET - self.is_venv = not os.path.samefile(os.path.abspath(prefix), sys.prefix) + self.is_venv = not os.path.samefile(os.path.abspath(str(prefix)), sys.prefix) if not sources: sources = [] self.sources = sources From 0c7f287ec389b4253d25a53d4bc36973ea47d92f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 13:36:58 -0500 Subject: [PATCH 164/218] Fix prefix comparison for py2 Signed-off-by: Dan Ryan --- pipenv/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index c3c33dda..b96f2fbe 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -26,7 +26,7 @@ class Environment(object): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET - self.is_venv = not os.path.samefile(os.path.abspath(prefix), sys.prefix) + self.is_venv = not os.path.samefile(os.path.abspath(str(prefix)), sys.prefix) if not sources: sources = [] self.sources = sources From d7d50ef3b745c822fd4b3aff509718fc3b57f40f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 16:31:53 -0500 Subject: [PATCH 165/218] no samefile for windows python2.7 Signed-off-by: Dan Ryan --- pipenv/environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index b96f2fbe..9315447c 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -26,7 +26,8 @@ class Environment(object): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET - self.is_venv = not os.path.samefile(os.path.abspath(str(prefix)), sys.prefix) + prefix = os.path.normcase(os.path.normpath(os.path.abspath(str(prefix)))) + self.is_venv = not prefix == os.path.normcase(os.path.normpath(sys.prefix)) if not sources: sources = [] self.sources = sources From 9eabde0bbfa3dc61d71fc1efbc741047dc417663 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 16:31:53 -0500 Subject: [PATCH 166/218] no samefile for windows python2.7 Signed-off-by: Dan Ryan --- pipenv/environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index b96f2fbe..9315447c 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -26,7 +26,8 @@ class Environment(object): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET - self.is_venv = not os.path.samefile(os.path.abspath(str(prefix)), sys.prefix) + prefix = os.path.normcase(os.path.normpath(os.path.abspath(str(prefix)))) + self.is_venv = not prefix == os.path.normcase(os.path.normpath(sys.prefix)) if not sources: sources = [] self.sources = sources From ef59d1520e9b84bd611587a2fd79680df19d53d9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 22:27:38 -0500 Subject: [PATCH 167/218] Fix bugs in environment implementation - Fix virtualenv - Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/core.py | 5 +- pipenv/environment.py | 33 +++++----- pipenv/project.py | 5 +- pipenv/vendor/pythonfinder/environment.py | 4 ++ pipenv/vendor/pythonfinder/models/asdf.py | 9 +++ pipenv/vendor/pythonfinder/models/path.py | 69 +++++++++++++++------ pipenv/vendor/pythonfinder/models/pyenv.py | 7 ++- pipenv/vendor/pythonfinder/models/python.py | 2 +- pipenv/vendor/pythonfinder/utils.py | 6 +- 9 files changed, 96 insertions(+), 44 deletions(-) create mode 100644 pipenv/vendor/pythonfinder/models/asdf.py diff --git a/pipenv/core.py b/pipenv/core.py index c2df7b78..6576648f 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -921,7 +921,8 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): prefix=project.get_location_for_virtualenv(), is_venv=True, sources=sources, - pipfile=project.parsed_pipfile + pipfile=project.parsed_pipfile, + project=project ) project._environment.add_dist("pipenv") # Say where the virtualenv is. @@ -1621,7 +1622,7 @@ def do_outdated(pypi_mirror=None): outdated_packages = { canonicalize_name(pkg.project_name): package_info (pkg.project_name, pkg.parsed_version, pkg.latest_version) - for pkg in project.get_outdated_packages() + for pkg in project.environment.get_outdated_packages() } for result in installed_packages: dep = Requirement.from_line(str(result.as_requirement())) diff --git a/pipenv/environment.py b/pipenv/environment.py index 9315447c..8548c38f 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -22,7 +22,8 @@ BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path) class Environment(object): - def __init__(self, prefix=None, is_venv=False, base_working_set=None, pipfile=None, sources=None): + def __init__(self, prefix=None, is_venv=False, base_working_set=None, pipfile=None, + sources=None, project=None): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET @@ -30,10 +31,17 @@ class Environment(object): self.is_venv = not prefix == os.path.normcase(os.path.normpath(sys.prefix)) if not sources: sources = [] + self.project = project + if project and not sources: + sources = project.sources self.sources = sources + if project and not pipfile: + pipfile = project.pipfile + self.pipfile = pipfile self.extra_dists = [] prefix = prefix if prefix else sys.prefix self.prefix = vistir.compat.Path(prefix) + self.sys_paths = get_paths() def safe_import(self, name): """Helper utility for reimporting previously imported modules while inside the env""" @@ -73,7 +81,7 @@ class Environment(object): deps.add(dist) try: reqs = dist.requires() - except AttributeError: + except (AttributeError, OSError): # The METADATA file can't be found return deps for req in reqs: dist = working_set.find(req) @@ -187,12 +195,6 @@ class Environment(object): path = json.loads(path.strip()) return path - @cached_property - def system_paths(self): - paths = {} - paths = get_paths() - return paths - @cached_property def sys_prefix(self): """The prefix run inside the context of the environment @@ -271,7 +273,8 @@ class Environment(object): packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] return packages - def get_finder(self): + @contextlib.contextmanager + def get_finder(self, pre=False): from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder from .environments import PIPENV_CACHE_DIR index_urls = [source.get("url") for source in self.sources] @@ -286,10 +289,10 @@ class Environment(object): cmd_opts = pip_command.cmd_opts pip_command.parser.insert_option_group(0, index_opts) pip_command.parser.insert_option_group(0, cmd_opts) - pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources, []) + pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources) pip_options, _ = pip_command.parser.parse_args(pip_args) pip_options.cache_dir = PIPENV_CACHE_DIR - pip_options.pre = self.pipfile.get("pre", False) + pip_options.pre = self.pipfile.get("pre", pre) with pip_command._build_session(pip_options) as session: finder = PackageFinder( find_links=pip_options.find_links, @@ -300,7 +303,7 @@ class Environment(object): ) yield finder - def get_package_info(self): + def get_package_info(self, pre=False): dependency_links = [] packages = self.get_installed_packages() # This code is borrowed from pip's current implementation @@ -314,7 +317,7 @@ class Environment(object): for dist in packages: typ = 'unknown' all_candidates = finder.find_all_candidates(dist.key) - if not finder.pip_options.pre: + if not self.pipfile.get("pre", finder.allow_all_prereleases): # Remove prereleases all_candidates = [ candidate for candidate in all_candidates @@ -334,9 +337,9 @@ class Environment(object): dist.latest_filetype = typ yield dist - def get_outdated_packages(self): + def get_outdated_packages(self, pre=False): return [ - pkg for pkg in self.get_package_info() + pkg for pkg in self.get_package_info(pre=pre) if pkg.latest_version._version > pkg.parsed_version._version ] diff --git a/pipenv/project.py b/pipenv/project.py index 0eafff8e..7857b25a 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -352,13 +352,14 @@ class Project(object): is_venv = prefix == sys.prefix sources = self.sources if self.sources else [DEFAULT_SOURCE,] self._environment = Environment( - prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile + prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile, + project=self ) self._environment.add_dist("pipenv") return self._environment def get_outdated_packages(self): - return self.environment.get_outdated_packages() + return self.environment.get_outdated_packages(pre=self.pipfile.get("pre", False)) @classmethod def _sanitize(cls, name): diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index 27a5b3fc..ec4a760f 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -7,9 +7,13 @@ import sys PYENV_INSTALLED = bool(os.environ.get("PYENV_SHELL")) or bool( os.environ.get("PYENV_ROOT") ) +ASDF_INSTALLED = bool(os.environ.get("ASDF_DATA_DIR")) PYENV_ROOT = os.path.expanduser( os.path.expandvars(os.environ.get("PYENV_ROOT", "~/.pyenv")) ) +ASDF_DATA_DIR = os.path.expanduser( + os.path.expandvars(os.environ.get("ASDF_DATA_DIR", "~/.asdf")) +) IS_64BIT_OS = None SYSTEM_ARCH = platform.architecture()[0] diff --git a/pipenv/vendor/pythonfinder/models/asdf.py b/pipenv/vendor/pythonfinder/models/asdf.py new file mode 100644 index 00000000..3ba6e4fa --- /dev/null +++ b/pipenv/vendor/pythonfinder/models/asdf.py @@ -0,0 +1,9 @@ +# -*- coding=utf-8 -*- +import attr + +from .pyenv import PyenvFinder + + +@attr.s +class AsdfFinder(PyenvFinder): + version_root = attr.ib(default="installs/python/*") diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 3d01e7cf..9c96e5f8 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -17,7 +17,7 @@ from cached_property import cached_property from vistir.compat import Path, fs_str from .mixins import BasePath -from ..environment import PYENV_INSTALLED, PYENV_ROOT +from ..environment import PYENV_INSTALLED, PYENV_ROOT, ASDF_INSTALLED, ASDF_DATA_DIR from ..exceptions import InvalidPythonVersion from ..utils import ( ensure_path, @@ -40,6 +40,7 @@ class SystemPath(object): python_version_dict = attr.ib(default=attr.Factory(defaultdict)) only_python = attr.ib(default=False) pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) + asdf_finder = attr.ib(default=None) system = attr.ib(default=False) _version_dict = attr.ib(default=attr.Factory(defaultdict)) ignore_unsupported = attr.ib(default=False) @@ -105,6 +106,8 @@ class SystemPath(object): self._setup_windows() if PYENV_INSTALLED: self._setup_pyenv() + if ASDF_INSTALLED: + self._setup_asdf() venv = os.environ.get("VIRTUAL_ENV") if os.name == "nt": bin_dir = "Scripts" @@ -124,32 +127,62 @@ class SystemPath(object): path=syspath_bin, is_root=True, only_python=False ) - def _setup_pyenv(self): - from .pyenv import PyenvFinder - - last_pyenv = next( - (p for p in reversed(self.path_order) if PYENV_ROOT.lower() in p.lower()), + def _get_last_instance(self, path): + last_instance = next(iter( + (p for p in reversed(self.path_order) if path.lower() in p.lower())), None, ) try: - pyenv_index = self.path_order.index(last_pyenv) + path_index = self.path_order.index(last_instance) except ValueError: return + return path_index + + def _slice_in_paths(self, start_idx, paths): + before_path = self.path_order[: start_idx + 1] + after_path = self.path_order[start_idx + 2 :] + self.path_order = ( + before_path + [p.as_posix() for p in paths] + after_path + ) + + def _remove_path(self, path): + path_copy = reversed(self.path_order[:]) + new_order = [] + target = os.path.normcase(os.path.normpath(os.path.abspath(path))) + path_map = { + os.path.normcase(os.path.normpath(os.path.abspath(pth))): pth + for pth in self.paths.keys() + } + if target in path_map: + del self.paths[path_map.get(target)] + for current_path in path_copy: + normalized = os.path.normcase(os.path.normpath(os.path.abspath(current_path))) + if normalized != target: + new_order.append(normalized) + new_order = reversed(new_order) + self.path_order = new_order + + def _setup_asdf(self): + from .asdf import AsdfFinder + asdf_index = self._get_last_instance(ASDF_DATA_DIR) + self.asdf_finder = AsdfFinder.create(root=ASDF_DATA_DIR, ignore_unsupported=True) + root_paths = [p for p in self.asdf_finder.roots] + self._slice_in_paths(asdf_index, root_paths) + self.paths.update(self.asdf_finder.roots) + self._register_finder("asdf", self.asdf_finder) + + def _setup_pyenv(self): + from .pyenv import PyenvFinder + + pyenv_index = self._get_last_instance(PYENV_ROOT) self.pyenv_finder = PyenvFinder.create( root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported ) root_paths = [p for p in self.pyenv_finder.roots] - before_path = self.path_order[: pyenv_index + 1] - after_path = self.path_order[pyenv_index + 2 :] - self.path_order = ( - before_path + [p.as_posix() for p in root_paths] + after_path - ) - pyenv_shim_path = os.path.join(PYENV_ROOT, "shims") - if pyenv_shim_path in self.path_order: - self.path_order.remove(pyenv_shim_path) + self._slice_in_paths(pyenv_index, root_paths) + self.paths.update(self.pyenv_finder.roots) - if pyenv_shim_path in self.paths: - del self.paths[pyenv_shim_path] + self._remove_path(os.path.join(PYENV_ROOT, "shims")) self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): @@ -396,7 +429,7 @@ class SystemPath(object): ) -@attr.s +@attr.s(slots=True) class PathEntry(BasePath): path = attr.ib(default=None, validator=optional_instance_of(Path)) _children = attr.ib(default=attr.Factory(dict)) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index ac7f8588..cf85f57a 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -26,7 +26,7 @@ from .python import PythonVersion logger = logging.getLogger(__name__) -@attr.s +@attr.s(slots=True) class PyenvFinder(BaseFinder, BasePath): root = attr.ib(default=None, validator=optional_instance_of(Path)) #: ignore_unsupported should come before versions, because its value is used @@ -34,6 +34,7 @@ class PyenvFinder(BaseFinder, BasePath): ignore_unsupported = attr.ib(default=True) paths = attr.ib(default=attr.Factory(list)) roots = attr.ib(default=attr.Factory(defaultdict)) + version_root = attr.ib(default="versions/*") versions = attr.ib() pythons = attr.ib() @@ -50,7 +51,7 @@ class PyenvFinder(BaseFinder, BasePath): version_order_lines = version_order_file.read_text(encoding="utf-8").splitlines() version_paths = [ - p for p in self.root.glob("versions/*") + p for p in self.root.glob(self.version_root) if not (p.parent.name == "envs" or p.name == "envs") ] versions = {v.name: v for v in version_paths} @@ -74,7 +75,7 @@ class PyenvFinder(BaseFinder, BasePath): @versions.default def get_versions(self): versions = defaultdict() - bin_ = sysconfig._INSTALL_SCHEMES['posix_prefix']["scripts"] + bin_ = "{base}/bin" for p in self.get_version_order(): bin_dir = Path(bin_.format(base=p.as_posix())) version_path = None diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 24d520b6..583dc6b3 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -21,7 +21,7 @@ from ..utils import ( ) -@attr.s +@attr.s(slots=True) class PythonVersion(object): major = attr.ib(default=0) minor = attr.ib(default=None) diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 42a63e54..881cdb2e 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -54,7 +54,7 @@ def get_python_version(path): version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, - combine_stderr=False) + combine_stderr=False) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) if not c.out: @@ -92,7 +92,7 @@ def looks_like_python(name): @lru_cache(maxsize=1024) def path_is_python(path): - return path_is_executable(path) and looks_like_python(path.name) + return path_is_known_executable(path) and looks_like_python(path.name) @lru_cache(maxsize=1024) @@ -117,7 +117,7 @@ def _filter_none(k, v): return False -@lru_cache(maxsize=128) +@lru_cache(maxsize=1024) def filter_pythons(path): """Return all valid pythons in a given path""" if not isinstance(path, vistir.compat.Path): From 9296f561d91f847ad65279b41c458d41a67f7ff7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 22:27:38 -0500 Subject: [PATCH 168/218] Fix bugs in environment implementation - Fix virtualenv - Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/core.py | 5 +- pipenv/environment.py | 33 +++++----- pipenv/project.py | 5 +- pipenv/vendor/pythonfinder/environment.py | 4 ++ pipenv/vendor/pythonfinder/models/asdf.py | 9 +++ pipenv/vendor/pythonfinder/models/path.py | 69 +++++++++++++++------ pipenv/vendor/pythonfinder/models/pyenv.py | 7 ++- pipenv/vendor/pythonfinder/models/python.py | 2 +- pipenv/vendor/pythonfinder/utils.py | 6 +- 9 files changed, 96 insertions(+), 44 deletions(-) create mode 100644 pipenv/vendor/pythonfinder/models/asdf.py diff --git a/pipenv/core.py b/pipenv/core.py index ee1a7df8..8504553c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -926,7 +926,8 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): prefix=project.get_location_for_virtualenv(), is_venv=True, sources=sources, - pipfile=project.parsed_pipfile + pipfile=project.parsed_pipfile, + project=project ) project._environment.add_dist("pipenv") # Say where the virtualenv is. @@ -1626,7 +1627,7 @@ def do_outdated(pypi_mirror=None): outdated_packages = { canonicalize_name(pkg.project_name): package_info (pkg.project_name, pkg.parsed_version, pkg.latest_version) - for pkg in project.get_outdated_packages() + for pkg in project.environment.get_outdated_packages() } for result in installed_packages: dep = Requirement.from_line(str(result.as_requirement())) diff --git a/pipenv/environment.py b/pipenv/environment.py index 9315447c..8548c38f 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -22,7 +22,8 @@ BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path) class Environment(object): - def __init__(self, prefix=None, is_venv=False, base_working_set=None, pipfile=None, sources=None): + def __init__(self, prefix=None, is_venv=False, base_working_set=None, pipfile=None, + sources=None, project=None): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET @@ -30,10 +31,17 @@ class Environment(object): self.is_venv = not prefix == os.path.normcase(os.path.normpath(sys.prefix)) if not sources: sources = [] + self.project = project + if project and not sources: + sources = project.sources self.sources = sources + if project and not pipfile: + pipfile = project.pipfile + self.pipfile = pipfile self.extra_dists = [] prefix = prefix if prefix else sys.prefix self.prefix = vistir.compat.Path(prefix) + self.sys_paths = get_paths() def safe_import(self, name): """Helper utility for reimporting previously imported modules while inside the env""" @@ -73,7 +81,7 @@ class Environment(object): deps.add(dist) try: reqs = dist.requires() - except AttributeError: + except (AttributeError, OSError): # The METADATA file can't be found return deps for req in reqs: dist = working_set.find(req) @@ -187,12 +195,6 @@ class Environment(object): path = json.loads(path.strip()) return path - @cached_property - def system_paths(self): - paths = {} - paths = get_paths() - return paths - @cached_property def sys_prefix(self): """The prefix run inside the context of the environment @@ -271,7 +273,8 @@ class Environment(object): packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] return packages - def get_finder(self): + @contextlib.contextmanager + def get_finder(self, pre=False): from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder from .environments import PIPENV_CACHE_DIR index_urls = [source.get("url") for source in self.sources] @@ -286,10 +289,10 @@ class Environment(object): cmd_opts = pip_command.cmd_opts pip_command.parser.insert_option_group(0, index_opts) pip_command.parser.insert_option_group(0, cmd_opts) - pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources, []) + pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources) pip_options, _ = pip_command.parser.parse_args(pip_args) pip_options.cache_dir = PIPENV_CACHE_DIR - pip_options.pre = self.pipfile.get("pre", False) + pip_options.pre = self.pipfile.get("pre", pre) with pip_command._build_session(pip_options) as session: finder = PackageFinder( find_links=pip_options.find_links, @@ -300,7 +303,7 @@ class Environment(object): ) yield finder - def get_package_info(self): + def get_package_info(self, pre=False): dependency_links = [] packages = self.get_installed_packages() # This code is borrowed from pip's current implementation @@ -314,7 +317,7 @@ class Environment(object): for dist in packages: typ = 'unknown' all_candidates = finder.find_all_candidates(dist.key) - if not finder.pip_options.pre: + if not self.pipfile.get("pre", finder.allow_all_prereleases): # Remove prereleases all_candidates = [ candidate for candidate in all_candidates @@ -334,9 +337,9 @@ class Environment(object): dist.latest_filetype = typ yield dist - def get_outdated_packages(self): + def get_outdated_packages(self, pre=False): return [ - pkg for pkg in self.get_package_info() + pkg for pkg in self.get_package_info(pre=pre) if pkg.latest_version._version > pkg.parsed_version._version ] diff --git a/pipenv/project.py b/pipenv/project.py index 0eafff8e..7857b25a 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -352,13 +352,14 @@ class Project(object): is_venv = prefix == sys.prefix sources = self.sources if self.sources else [DEFAULT_SOURCE,] self._environment = Environment( - prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile + prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile, + project=self ) self._environment.add_dist("pipenv") return self._environment def get_outdated_packages(self): - return self.environment.get_outdated_packages() + return self.environment.get_outdated_packages(pre=self.pipfile.get("pre", False)) @classmethod def _sanitize(cls, name): diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index 27a5b3fc..ec4a760f 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -7,9 +7,13 @@ import sys PYENV_INSTALLED = bool(os.environ.get("PYENV_SHELL")) or bool( os.environ.get("PYENV_ROOT") ) +ASDF_INSTALLED = bool(os.environ.get("ASDF_DATA_DIR")) PYENV_ROOT = os.path.expanduser( os.path.expandvars(os.environ.get("PYENV_ROOT", "~/.pyenv")) ) +ASDF_DATA_DIR = os.path.expanduser( + os.path.expandvars(os.environ.get("ASDF_DATA_DIR", "~/.asdf")) +) IS_64BIT_OS = None SYSTEM_ARCH = platform.architecture()[0] diff --git a/pipenv/vendor/pythonfinder/models/asdf.py b/pipenv/vendor/pythonfinder/models/asdf.py new file mode 100644 index 00000000..3ba6e4fa --- /dev/null +++ b/pipenv/vendor/pythonfinder/models/asdf.py @@ -0,0 +1,9 @@ +# -*- coding=utf-8 -*- +import attr + +from .pyenv import PyenvFinder + + +@attr.s +class AsdfFinder(PyenvFinder): + version_root = attr.ib(default="installs/python/*") diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 3d01e7cf..9c96e5f8 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -17,7 +17,7 @@ from cached_property import cached_property from vistir.compat import Path, fs_str from .mixins import BasePath -from ..environment import PYENV_INSTALLED, PYENV_ROOT +from ..environment import PYENV_INSTALLED, PYENV_ROOT, ASDF_INSTALLED, ASDF_DATA_DIR from ..exceptions import InvalidPythonVersion from ..utils import ( ensure_path, @@ -40,6 +40,7 @@ class SystemPath(object): python_version_dict = attr.ib(default=attr.Factory(defaultdict)) only_python = attr.ib(default=False) pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) + asdf_finder = attr.ib(default=None) system = attr.ib(default=False) _version_dict = attr.ib(default=attr.Factory(defaultdict)) ignore_unsupported = attr.ib(default=False) @@ -105,6 +106,8 @@ class SystemPath(object): self._setup_windows() if PYENV_INSTALLED: self._setup_pyenv() + if ASDF_INSTALLED: + self._setup_asdf() venv = os.environ.get("VIRTUAL_ENV") if os.name == "nt": bin_dir = "Scripts" @@ -124,32 +127,62 @@ class SystemPath(object): path=syspath_bin, is_root=True, only_python=False ) - def _setup_pyenv(self): - from .pyenv import PyenvFinder - - last_pyenv = next( - (p for p in reversed(self.path_order) if PYENV_ROOT.lower() in p.lower()), + def _get_last_instance(self, path): + last_instance = next(iter( + (p for p in reversed(self.path_order) if path.lower() in p.lower())), None, ) try: - pyenv_index = self.path_order.index(last_pyenv) + path_index = self.path_order.index(last_instance) except ValueError: return + return path_index + + def _slice_in_paths(self, start_idx, paths): + before_path = self.path_order[: start_idx + 1] + after_path = self.path_order[start_idx + 2 :] + self.path_order = ( + before_path + [p.as_posix() for p in paths] + after_path + ) + + def _remove_path(self, path): + path_copy = reversed(self.path_order[:]) + new_order = [] + target = os.path.normcase(os.path.normpath(os.path.abspath(path))) + path_map = { + os.path.normcase(os.path.normpath(os.path.abspath(pth))): pth + for pth in self.paths.keys() + } + if target in path_map: + del self.paths[path_map.get(target)] + for current_path in path_copy: + normalized = os.path.normcase(os.path.normpath(os.path.abspath(current_path))) + if normalized != target: + new_order.append(normalized) + new_order = reversed(new_order) + self.path_order = new_order + + def _setup_asdf(self): + from .asdf import AsdfFinder + asdf_index = self._get_last_instance(ASDF_DATA_DIR) + self.asdf_finder = AsdfFinder.create(root=ASDF_DATA_DIR, ignore_unsupported=True) + root_paths = [p for p in self.asdf_finder.roots] + self._slice_in_paths(asdf_index, root_paths) + self.paths.update(self.asdf_finder.roots) + self._register_finder("asdf", self.asdf_finder) + + def _setup_pyenv(self): + from .pyenv import PyenvFinder + + pyenv_index = self._get_last_instance(PYENV_ROOT) self.pyenv_finder = PyenvFinder.create( root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported ) root_paths = [p for p in self.pyenv_finder.roots] - before_path = self.path_order[: pyenv_index + 1] - after_path = self.path_order[pyenv_index + 2 :] - self.path_order = ( - before_path + [p.as_posix() for p in root_paths] + after_path - ) - pyenv_shim_path = os.path.join(PYENV_ROOT, "shims") - if pyenv_shim_path in self.path_order: - self.path_order.remove(pyenv_shim_path) + self._slice_in_paths(pyenv_index, root_paths) + self.paths.update(self.pyenv_finder.roots) - if pyenv_shim_path in self.paths: - del self.paths[pyenv_shim_path] + self._remove_path(os.path.join(PYENV_ROOT, "shims")) self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): @@ -396,7 +429,7 @@ class SystemPath(object): ) -@attr.s +@attr.s(slots=True) class PathEntry(BasePath): path = attr.ib(default=None, validator=optional_instance_of(Path)) _children = attr.ib(default=attr.Factory(dict)) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index ac7f8588..cf85f57a 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -26,7 +26,7 @@ from .python import PythonVersion logger = logging.getLogger(__name__) -@attr.s +@attr.s(slots=True) class PyenvFinder(BaseFinder, BasePath): root = attr.ib(default=None, validator=optional_instance_of(Path)) #: ignore_unsupported should come before versions, because its value is used @@ -34,6 +34,7 @@ class PyenvFinder(BaseFinder, BasePath): ignore_unsupported = attr.ib(default=True) paths = attr.ib(default=attr.Factory(list)) roots = attr.ib(default=attr.Factory(defaultdict)) + version_root = attr.ib(default="versions/*") versions = attr.ib() pythons = attr.ib() @@ -50,7 +51,7 @@ class PyenvFinder(BaseFinder, BasePath): version_order_lines = version_order_file.read_text(encoding="utf-8").splitlines() version_paths = [ - p for p in self.root.glob("versions/*") + p for p in self.root.glob(self.version_root) if not (p.parent.name == "envs" or p.name == "envs") ] versions = {v.name: v for v in version_paths} @@ -74,7 +75,7 @@ class PyenvFinder(BaseFinder, BasePath): @versions.default def get_versions(self): versions = defaultdict() - bin_ = sysconfig._INSTALL_SCHEMES['posix_prefix']["scripts"] + bin_ = "{base}/bin" for p in self.get_version_order(): bin_dir = Path(bin_.format(base=p.as_posix())) version_path = None diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 24d520b6..583dc6b3 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -21,7 +21,7 @@ from ..utils import ( ) -@attr.s +@attr.s(slots=True) class PythonVersion(object): major = attr.ib(default=0) minor = attr.ib(default=None) diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 42a63e54..881cdb2e 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -54,7 +54,7 @@ def get_python_version(path): version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, - combine_stderr=False) + combine_stderr=False) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) if not c.out: @@ -92,7 +92,7 @@ def looks_like_python(name): @lru_cache(maxsize=1024) def path_is_python(path): - return path_is_executable(path) and looks_like_python(path.name) + return path_is_known_executable(path) and looks_like_python(path.name) @lru_cache(maxsize=1024) @@ -117,7 +117,7 @@ def _filter_none(k, v): return False -@lru_cache(maxsize=128) +@lru_cache(maxsize=1024) def filter_pythons(path): """Return all valid pythons in a given path""" if not isinstance(path, vistir.compat.Path): From f69b79645b2d346d0458a4c08139910476c2ac97 Mon Sep 17 00:00:00 2001 From: frostming Date: Tue, 13 Nov 2018 11:45:19 +0800 Subject: [PATCH 169/218] Bring back the fix of #3147 Signed-off-by: frostming --- pipenv/utils.py | 60 ++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 8674aee2..3e749ee5 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -361,36 +361,6 @@ class Resolver(object): clear_caches=clear, prereleases=pre, ) - def populate_file_hashes(self): - from pipenv.vendor.vistir.compat import Path, to_native_string - from pipenv.vendor.vistir.path import url_to_path - - def _should_include(ireq): - # We can only hash artifacts. - try: - if not ireq.link.is_artifact: - return False - except AttributeError: - return False - - # But we don't want normal pypi artifcats since the normal resolver - # handles those - if is_pypi_url(ireq.link.url): - return False - - # We also don't want to try to hash directories as this will fail - # as these are editable deps and are not hashable. - if (ireq.link.scheme == "file" and - Path(to_native_string(url_to_path(ireq.link.url))).is_dir()): - return False - return True - - self.hashes.update({ - ireq: self.resolver._hash_cache.get_hash(ireq.link) - for ireq in self.parsed_constraints - if _should_include(ireq) - }) - @property def resolver(self): if self._resolver is None: @@ -421,11 +391,39 @@ class Resolver(object): return self.resolved_tree def resolve_hashes(self): + def _should_include_hash(ireq): + from pipenv.vendor.vistir.compat import Path, to_native_string + from pipenv.vendor.vistir.path import url_to_path + + # We can only hash artifacts. + try: + if not ireq.link.is_artifact: + return False + except AttributeError: + return False + + # But we don't want normal pypi artifcats since the normal resolver + # handles those + if is_pypi_url(ireq.link.url): + return False + + # We also don't want to try to hash directories as this will fail + # as these are editable deps and are not hashable. + if (ireq.link.scheme == "file" and + Path(to_native_string(url_to_path(ireq.link.url))).is_dir()): + return False + return True + if self.results is not None: resolved_hashes = self.resolver.resolve_hashes(self.results) for ireq, ireq_hashes in resolved_hashes.items(): if ireq not in self.hashes: - self.hashes[ireq] = ireq_hashes + if _should_include_hash(ireq): + self.hashes[ireq] = [ + self.resolver.repository._hash_cache.get_hash(ireq.link) + ] + else: + self.hashes[ireq] = ireq_hashes return self.hashes From e62b8005068c3a12401c2a820d370e854ecdd6f7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 23:22:48 -0500 Subject: [PATCH 170/218] Fix syntax Signed-off-by: Dan Ryan --- pipenv/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index 8548c38f..8e96f2c8 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -36,7 +36,7 @@ class Environment(object): sources = project.sources self.sources = sources if project and not pipfile: - pipfile = project.pipfile + pipfile = project.parsed_pipfile self.pipfile = pipfile self.extra_dists = [] prefix = prefix if prefix else sys.prefix From cb601b0e5b67ec76d47b9a256e328da93bdb7574 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 23:22:48 -0500 Subject: [PATCH 171/218] Fix syntax Signed-off-by: Dan Ryan --- pipenv/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index 8548c38f..8e96f2c8 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -36,7 +36,7 @@ class Environment(object): sources = project.sources self.sources = sources if project and not pipfile: - pipfile = project.pipfile + pipfile = project.parsed_pipfile self.pipfile = pipfile self.extra_dists = [] prefix = prefix if prefix else sys.prefix From 13c9e62029184b8b30de3afdc7b51b1cae9da062 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 13 Nov 2018 00:39:20 -0500 Subject: [PATCH 172/218] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 21 +++++++++++++++++---- pipenv/vendor/pythonfinder/utils.py | 12 ++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 9c96e5f8..523f117a 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -26,6 +26,7 @@ from ..utils import ( optional_instance_of, path_is_known_executable, unnest, + normalize_path ) from .python import PythonVersion @@ -128,9 +129,10 @@ class SystemPath(object): ) def _get_last_instance(self, path): - last_instance = next(iter( - (p for p in reversed(self.path_order) if path.lower() in p.lower())), - None, + paths = [normalize_path(p) for p in reversed(self.path_order)] + normalized_target = normalize_path(path) + last_instance = next( + iter(p for p in paths if normalized_target in p), None ) try: path_index = self.path_order.index(last_instance) @@ -165,6 +167,10 @@ class SystemPath(object): def _setup_asdf(self): from .asdf import AsdfFinder asdf_index = self._get_last_instance(ASDF_DATA_DIR) + if not asdf_index: + # we are in a virtualenv without global pyenv on the path, so we should + # not write pyenv to the path here + return self.asdf_finder = AsdfFinder.create(root=ASDF_DATA_DIR, ignore_unsupported=True) root_paths = [p for p in self.asdf_finder.roots] self._slice_in_paths(asdf_index, root_paths) @@ -174,10 +180,14 @@ class SystemPath(object): def _setup_pyenv(self): from .pyenv import PyenvFinder - pyenv_index = self._get_last_instance(PYENV_ROOT) self.pyenv_finder = PyenvFinder.create( root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported ) + pyenv_index = self._get_last_instance(PYENV_ROOT) + if not pyenv_index: + # we are in a virtualenv without global pyenv on the path, so we should + # not write pyenv to the path here + return root_paths = [p for p in self.pyenv_finder.roots] self._slice_in_paths(pyenv_index, root_paths) @@ -485,6 +495,9 @@ class PathEntry(BasePath): py_version = PythonVersion.from_path(path=self, name=self.name) except InvalidPythonVersion: py_version = None + except Exception: + if not IGNORE_UNSUPPORTED: + raise return py_version return diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 881cdb2e..b8714f52 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -2,13 +2,9 @@ from __future__ import absolute_import, print_function import itertools -import locale import os -import subprocess -import sys from fnmatch import fnmatch -from itertools import chain import attr import six @@ -54,7 +50,7 @@ def get_python_version(path): version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, - combine_stderr=False) + combine_stderr=False) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) if not c.out: @@ -92,7 +88,7 @@ def looks_like_python(name): @lru_cache(maxsize=1024) def path_is_python(path): - return path_is_known_executable(path) and looks_like_python(path.name) + return path_is_executable(path) and looks_like_python(path.name) @lru_cache(maxsize=1024) @@ -117,6 +113,10 @@ def _filter_none(k, v): return False +def normalize_path(path): + return os.path.normpath(os.path.normcase(os.path.abspath(path))) + + @lru_cache(maxsize=1024) def filter_pythons(path): """Return all valid pythons in a given path""" From d73879b5e57773973fdeb362d97bef7f46762046 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 13 Nov 2018 01:01:49 -0500 Subject: [PATCH 173/218] Update requirementslib - Fix ref parsing - Fixes #3214 Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/__init__.py | 2 +- pipenv/vendor/requirementslib/models/requirements.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 05fd1943..f6c985d3 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.2.6' +__version__ = '1.2.7' import logging import warnings diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index d5330b48..d034a12d 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -891,8 +891,8 @@ class VCSRequirement(FileRequirement): name = link.egg_fragment subdirectory = link.subdirectory_fragment ref = None - if "@" in link.show_url and "@" in uri: - uri, ref = uri.rsplit("@", 1) + if "@" in link.path and "@" in uri: + uri, _, ref = uri.rpartition("@") if relpath and "@" in relpath: relpath, ref = relpath.rsplit("@", 1) return cls( From e328ae24dfb61e5d207f1a64f85b9ea949092ed1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 13 Nov 2018 01:23:19 -0500 Subject: [PATCH 174/218] Fix feedback Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 6 +++--- pipenv/vendor/pythonfinder/utils.py | 2 +- pipenv/vendor/vendor.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 523f117a..df755fa6 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -150,15 +150,15 @@ class SystemPath(object): def _remove_path(self, path): path_copy = reversed(self.path_order[:]) new_order = [] - target = os.path.normcase(os.path.normpath(os.path.abspath(path))) + target = normalize_path(path) path_map = { - os.path.normcase(os.path.normpath(os.path.abspath(pth))): pth + normalize_path(pth): pth for pth in self.paths.keys() } if target in path_map: del self.paths[path_map.get(target)] for current_path in path_copy: - normalized = os.path.normcase(os.path.normpath(os.path.abspath(current_path))) + normalized = normalize_path(current_path) if normalized != target: new_order.append(normalized) new_order = reversed(new_order) diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index b8714f52..fb932b10 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -114,7 +114,7 @@ def _filter_none(k, v): def normalize_path(path): - return os.path.normpath(os.path.normcase(os.path.abspath(path))) + return os.path.normpath(os.path.normcase(os.path.abspath(str(path)))) @lru_cache(maxsize=1024) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index c106a59c..62317853 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -34,7 +34,7 @@ requirementslib==1.3.0 pyparsing==2.2.2 pytoml==0.1.19 plette==0.2.2 - tomlkit==0.5.2 + tomlkit==0.4.6 shellingham==1.2.7 six==1.11.0 semver==2.8.1 From 310e0b293bf5febd035430d2a2f80e5c3158c5c8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 13 Nov 2018 10:17:19 -0500 Subject: [PATCH 175/218] Fix pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 43 +++- pipenv/vendor/pythonfinder/models/pyenv.py | 2 - pipenv/vendor/pythonfinder/models/python.py | 221 +++++++++++++++++++- pipenv/vendor/pythonfinder/utils.py | 35 +++- 4 files changed, 284 insertions(+), 17 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index df755fa6..d3cdd9d1 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -26,7 +26,9 @@ from ..utils import ( optional_instance_of, path_is_known_executable, unnest, - normalize_path + normalize_path, + parse_pyenv_version_order, + parse_asdf_version_order ) from .python import PythonVersion @@ -165,23 +167,26 @@ class SystemPath(object): self.path_order = new_order def _setup_asdf(self): - from .asdf import AsdfFinder + from .python import PythonFinder asdf_index = self._get_last_instance(ASDF_DATA_DIR) if not asdf_index: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here return - self.asdf_finder = AsdfFinder.create(root=ASDF_DATA_DIR, ignore_unsupported=True) + self.asdf_finder = PythonFinder.create( + root=ASDF_DATA_DIR, ignore_unsupported=True, + sort_function=parse_asdf_version_order, version_glob_path="installs/python/*") root_paths = [p for p in self.asdf_finder.roots] self._slice_in_paths(asdf_index, root_paths) self.paths.update(self.asdf_finder.roots) self._register_finder("asdf", self.asdf_finder) def _setup_pyenv(self): - from .pyenv import PyenvFinder + from .python import PythonFinder - self.pyenv_finder = PyenvFinder.create( - root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported + self.pyenv_finder = PythonFinder.create( + root=PYENV_ROOT, sort_function=parse_pyenv_version_order, + version_glob_path="versions/*", ignore_unsupported=self.ignore_unsupported ) pyenv_index = self._get_last_instance(PYENV_ROOT) if not pyenv_index: @@ -585,3 +590,29 @@ class PathEntry(BasePath): return self.is_executable and ( looks_like_python(self.path.name) ) + + +@attr.s +class VersionPath(SystemPath): + base = attr.ib(default=None, validator=optional_instance_of(Path)) + name = attr.ib(default=None) + + @classmethod + def create(cls, path, only_python=True, pythons=None, name=None): + """Accepts a path to a base python version directory. + + Generates the version listings for it""" + from .path import PathEntry + path = ensure_path(path) + path_entries = defaultdict(PathEntry) + bin_ = "{base}/bin" + if path.as_posix().endswith(Path(bin_).name): + path = path.parent + bin_dir = ensure_path(bin_.format(base=path.as_posix())) + if not name: + name = path.name + current_entry = PathEntry.create( + bin_dir, is_root=True, only_python=True, pythons=pythons, name=name + ) + path_entries[bin_dir.as_posix()] = current_entry + return cls(name=name, base=bin_dir, paths=path_entries) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index cf85f57a..6f2d6422 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -14,8 +14,6 @@ from vistir.compat import Path from ..utils import ( ensure_path, optional_instance_of, - get_python_version, - filter_pythons, unnest, ) from .mixins import BaseFinder, BasePath diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 583dc6b3..7feee84e 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -3,23 +3,238 @@ from __future__ import absolute_import, print_function import copy import platform +import operator +import logging from collections import defaultdict import attr -from packaging.version import Version, LegacyVersion +from packaging.version import Version from packaging.version import parse as parse_version +from vistir.compat import Path -from ..environment import SYSTEM_ARCH +from ..environment import SYSTEM_ARCH, PYENV_ROOT, ASDF_DATA_DIR +from .mixins import BaseFinder, BasePath from ..utils import ( _filter_none, ensure_path, get_python_version, optional_instance_of, - ensure_path, + unnest, + is_in_path, + parse_pyenv_version_order, + parse_asdf_version_order, ) +logger = logging.getLogger(__name__) + + +@attr.s(slots=True) +class PythonFinder(BaseFinder, BasePath): + root = attr.ib(default=None, validator=optional_instance_of(Path)) + #: ignore_unsupported should come before versions, because its value is used + #: in versions's default initializer. + ignore_unsupported = attr.ib(default=True) + #: The function to use to sort version order when returning an ordered verion set + sort_function = attr.ib(default=None) + paths = attr.ib(default=attr.Factory(list)) + roots = attr.ib(default=attr.Factory(defaultdict)) + #: Glob path for python versions off of the root directory + version_glob_path = attr.ib(default="versions/*") + versions = attr.ib() + pythons = attr.ib() + + @property + def expanded_paths(self): + return ( + path for path in unnest(p for p in self.versions.values()) + if path is not None + ) + + @property + def is_pyenv(self): + return is_in_path(str(self.root), PYENV_ROOT) + + @property + def is_asdf(self): + return is_in_path(str(self.root), ASDF_DATA_DIR) + + def get_version_order(self): + version_paths = [ + p for p in self.root.glob(self.version_glob_path) + if not (p.parent.name == "envs" or p.name == "envs") + ] + versions = {v.name: v for v in version_paths} + if self.is_pyenv: + version_order = [versions[v] for v in parse_pyenv_version_order()] + elif self.is_asdf: + version_order = [versions[v] for v in parse_asdf_version_order()] + for version in version_order: + version_paths.remove(version) + if version_order: + version_order += version_paths + else: + version_order = version_paths + return version_order + + @classmethod + def version_from_bin_dir(cls, base_dir, name=None): + from .path import PathEntry + py_version = None + version_path = PathEntry.create( + path=base_dir.absolute().as_posix(), + only_python=True, + name=base_dir.parent.name, + ) + py_version = next(iter(version_path.find_all_python_versions()), None) + return py_version + + @versions.default + def get_versions(self): + from .path import PathEntry + versions = defaultdict() + bin_ = "{base}/bin" + for p in self.get_version_order(): + bin_dir = Path(bin_.format(base=p.as_posix())) + version_path = None + if bin_dir.exists(): + version_path = PathEntry.create( + path=bin_dir.absolute().as_posix(), + only_python=False, + name=p.name, + is_root=True, + ) + version = None + try: + version = PythonVersion.parse(p.name) + except ValueError: + entry = next(iter(version_path.find_all_python_versions()), None) + if not entry: + if self.ignore_unsupported: + continue + raise + else: + version = entry.py_version.as_dict() + except Exception: + if not self.ignore_unsupported: + raise + logger.warning( + "Unsupported Python version %r, ignoring...", p.name, exc_info=True + ) + continue + if not version: + continue + version_tuple = ( + version.get("major"), + version.get("minor"), + version.get("patch"), + version.get("is_prerelease"), + version.get("is_devrelease"), + version.get("is_debug"), + ) + self.roots[p] = version_path + versions[version_tuple] = version_path + self.paths.append(version_path) + return versions + + @pythons.default + def get_pythons(self): + pythons = defaultdict() + for p in self.paths: + pythons.update(p.pythons) + return pythons + + @classmethod + def create(cls, root, sort_function=None, version_glob_path=None, ignore_unsupported=True): + root = ensure_path(root) + if not version_glob_path: + version_glob_path = "versions/*" + return cls(root=root, ignore_unsupported=ignore_unsupported, + sort_function=sort_function, version_glob_path=version_glob_path) + + def find_all_python_versions( + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, + ): + """Search for a specific python version on the path. Return all copies + + :param major: Major python version to search for. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. + :rtype: List[:class:`~pythonfinder.models.PathEntry`] + """ + + version_matcher = operator.methodcaller( + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + ) + py = operator.attrgetter("as_python") + pythons = ( + py_ver for py_ver in (py(p) for p in self.pythons.values() if p is not None) + if py_ver is not None + ) + # pythons = filter(None, [p.as_python for p in self.pythons.values()]) + matching_versions = filter(lambda py: version_matcher(py), pythons) + version_sort = operator.attrgetter("version_sort") + return sorted(matching_versions, key=version_sort, reverse=True) + + def find_python_version( + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, + ): + """Search or self for the specified Python version and return the first match. + + :param major: Major version number. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. + """ + + version_matcher = operator.methodcaller( + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + ) + pythons = filter(None, [p.as_python for p in self.pythons.values()]) + matching_versions = filter(lambda py: version_matcher(py), pythons) + version_sort = operator.attrgetter("version_sort") + return next(iter(c for c in sorted(matching_versions, key=version_sort, reverse=True)), None) + @attr.s(slots=True) class PythonVersion(object): diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index fb932b10..9c71e380 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -7,10 +7,12 @@ import os from fnmatch import fnmatch import attr +import io import six import vistir +from .environment import PYENV_INSTALLED, PYENV_ROOT, ASDF_INSTALLED, ASDF_DATA_DIR from .exceptions import InvalidPythonVersion try: @@ -127,12 +129,6 @@ def filter_pythons(path): return filter(lambda x: path_is_python(x), path.iterdir()) -# def unnest(item): -# if isinstance(next((i for i in item), None), (list, tuple)): -# return chain(*filter(None, item)) -# return chain(filter(None, item)) - - def unnest(item): if isinstance(item, Iterable) and not isinstance(item, six.string_types): item, target = itertools.tee(item, 2) @@ -145,3 +141,30 @@ def unnest(item): yield sub else: yield el + + +def parse_pyenv_version_order(filename="version"): + version_order_file = normalize_path(os.path.join(PYENV_ROOT, filename)) + if os.path.exists(version_order_file) and os.path.isfile(version_order_file): + with io.open(version_order_file, encoding="utf-8") as fh: + contents = fh.read() + version_order = [v for v in contents.splitlines()] + return version_order + + +def parse_asdf_version_order(filename=".tool-versions"): + version_order_file = normalize_path(os.path.join("~", filename)) + if os.path.exists(version_order_file) and os.path.isfile(version_order_file): + with io.open(version_order_file, encoding="utf-8") as fh: + contents = fh.read() + python_section = next(iter( + line for line in contents.splitlines() if line.startswith("python") + ), None) + if python_section: + python_key, versions = python_section.partition() + if versions: + return versions.split() + + +def is_in_path(path, parent): + return normalize_path(str(path)).startswith(normalize_path(str(parent))) From 6b3c9a7eb79564daad03ba32aee318a6e8ce8195 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 13 Nov 2018 10:18:15 -0500 Subject: [PATCH 176/218] Remove accidentally committed test script Signed-off-by: Dan Ryan --- pipenv/test_script.py | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 pipenv/test_script.py diff --git a/pipenv/test_script.py b/pipenv/test_script.py deleted file mode 100644 index d599ded6..00000000 --- a/pipenv/test_script.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding=utf-8 -*- - -import os -import sys - - -def _patch_path(): - import site - pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) - pipenv_site_dir = os.path.dirname(pipenv_libdir) - site.addsitedir(pipenv_site_dir) - for _dir in ("vendor", "patched"): - sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) - - -def test_install(): - from pipenv.vendor.vistir.contextmanagers import cd - from pipenv.vendor.click.testing import CliRunner - runner = CliRunner() - with cd("/tmp/test"): - from pipenv.core import do_lock - locked = do_lock(system=False, clear=False, pre=False, keep_outdated=False, - write=True, pypi_mirror=None) - # result = runner.invoke(cli, ["lock", "--verbose"]) - # print(result.output) - # print(result.exit_code) - print(locked) - - -if __name__ == "__main__": - _patch_path() - test_install() From 72e2ef70c5b2d2b5820f5e7af1dc401e3da15cfb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 13 Nov 2018 14:56:26 -0500 Subject: [PATCH 177/218] Derive source names from URLs when not supplied - Fixes #3216 Signed-off-by: Dan Ryan --- news/3216.bugfix.rst | 1 + pipenv/environment.py | 15 ++++++++++++--- pipenv/project.py | 3 ++- pipenv/utils.py | 13 +++++++++++++ 4 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 news/3216.bugfix.rst diff --git a/news/3216.bugfix.rst b/news/3216.bugfix.rst new file mode 100644 index 00000000..1d600bb8 --- /dev/null +++ b/news/3216.bugfix.rst @@ -0,0 +1 @@ +When sources are missing names, names will now be derived from the supplied URL. diff --git a/pipenv/environment.py b/pipenv/environment.py index 8e96f2c8..7a7fd2ea 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -18,6 +18,8 @@ from cached_property import cached_property import vistir import pipenv +from .utils import normalize_path + BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path) @@ -27,8 +29,8 @@ class Environment(object): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET - prefix = os.path.normcase(os.path.normpath(os.path.abspath(str(prefix)))) - self.is_venv = not prefix == os.path.normcase(os.path.normpath(sys.prefix)) + prefix = normalize_path(prefix) + self.is_venv = not prefix == normalize_path(sys.prefix) if not sources: sources = [] self.project = project @@ -81,7 +83,7 @@ class Environment(object): deps.add(dist) try: reqs = dist.requires() - except (AttributeError, OSError): # The METADATA file can't be found + except (AttributeError, OSError, IOError): # The METADATA file can't be found return deps for req in reqs: dist = working_set.find(req) @@ -243,6 +245,7 @@ class Environment(object): return pkg_resources.find_distributions(self.paths["PYTHONPATH"]) def find_egg(self, egg_dist): + """Find an egg by name in the given environment""" site_packages = get_python_lib() search_filename = "{0}.egg-link".format(egg_dist.project_name) try: @@ -256,11 +259,16 @@ class Environment(object): return egg def locate_dist(self, dist): + """Given a distribution, try to find a corresponding egg link first. + + If the egg - link doesn 't exist, return the supplied distribution.""" + location = self.find_egg(dist) if not location: return dist.location def dist_is_in_project(self, dist): + """Determine whether the supplied distribution is in the environment.""" from .project import _normalized prefix = _normalized(self.base_paths["prefix"]) location = self.locate_dist(dist) @@ -269,6 +277,7 @@ class Environment(object): return _normalized(location).startswith(prefix) def get_installed_packages(self): + """Returns all of the installed packages in a given environment""" workingset = self.get_working_set() packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] return packages diff --git a/pipenv/project.py b/pipenv/project.py index 957b6827..590fafe5 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -29,6 +29,7 @@ from .utils import ( convert_toml_outline_tables, is_installable_file, is_valid_url, + get_url_name, normalize_drive, python_version, safe_expandvars, @@ -747,7 +748,7 @@ class Project(object): sources = [sources,] lockfile_dict["_meta"]["sources"] = [ { - "name": s["name"], + "name": s.get("name", get_url_name(s.get("url"))), "url": s["url"], "verify_ssl": ( s["verify_ssl"] if isinstance(s["verify_ssl"], bool) else ( diff --git a/pipenv/utils.py b/pipenv/utils.py index dcd67025..265c5797 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1133,6 +1133,19 @@ def path_to_url(path): return Path(normalize_drive(os.path.abspath(path))).as_uri() +def normalize_path(path): + return os.path.expandvars(os.path.expanduser( + os.path.normcase(os.path.normpath(os.path.abspath(str(path)))) + )) + + +def get_url_name(url): + if not isinstance(url, six.string_types): + return + from urllib3.util import parse as urllib3_parse + return urllib3_parse(url).host + + def get_canonical_names(packages): """Canonicalize a list of packages and return a set of canonical names""" from .vendor.packaging.utils import canonicalize_name From 1e3b8f961565b61640c7791d104c769534c0fcb4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 13 Nov 2018 15:16:53 -0500 Subject: [PATCH 178/218] Update news entries Signed-off-by: Dan Ryan --- news/2737.bugfix.rst | 2 +- news/2983.bugfix.rst | 2 +- news/3041.feature | 2 +- news/3061.vendor.rst | 1 + news/3088.bugfix.rst | 2 +- news/3096.bugfix.rst | 1 + news/3096.feature.rst | 1 + news/3096.vendor.rst | 6 ++++++ news/3145.bugfix.rst | 2 +- news/3170.feature | 2 +- ...-40ba-8242-1e6ed18fc2fe.feature.rst => 3217.feature.rst} | 0 11 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 news/3061.vendor.rst create mode 100644 news/3096.bugfix.rst create mode 100644 news/3096.feature.rst create mode 100644 news/3096.vendor.rst rename news/{d65e7c90-3e70-40ba-8242-1e6ed18fc2fe.feature.rst => 3217.feature.rst} (100%) diff --git a/news/2737.bugfix.rst b/news/2737.bugfix.rst index bddcff91..5716c8dc 100644 --- a/news/2737.bugfix.rst +++ b/news/2737.bugfix.rst @@ -1 +1 @@ -Handle non-ASCII characters correctly in TOML. +Non-ascii characters will now be handled correctly when parsed by pipenv's ``ToML`` parsers. diff --git a/news/2983.bugfix.rst b/news/2983.bugfix.rst index acfe3780..87dcd71f 100644 --- a/news/2983.bugfix.rst +++ b/news/2983.bugfix.rst @@ -1 +1 @@ -Pipenv will no longer fail when encountering python versions on Windows that were unintalled. +Pipenv will no longer fail when encountering python versions on Windows that have been uninstalled. diff --git a/news/3041.feature b/news/3041.feature index 79a1d5de..8a87b5f6 100644 --- a/news/3041.feature +++ b/news/3041.feature @@ -1 +1 @@ ---bare now has an effect on clean, and sync's bare option is now used to reduce output. +Added support for ``--bare`` to ``pipenv clean``, and fixed ``pipenv sync --bare`` to actually reduce output. diff --git a/news/3061.vendor.rst b/news/3061.vendor.rst new file mode 100644 index 00000000..fb815c42 --- /dev/null +++ b/news/3061.vendor.rst @@ -0,0 +1 @@ +Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. diff --git a/news/3088.bugfix.rst b/news/3088.bugfix.rst index b10c4b2b..fa57c42d 100644 --- a/news/3088.bugfix.rst +++ b/news/3088.bugfix.rst @@ -1 +1 @@ -Fixed a bug which caused ``Unexpected EOF`` errors to be thrown when PIP awaited input from users who put login credentials in their environment. +Fixed a bug which caused ``Unexpected EOF`` errors to be thrown when ``pip`` was waiting for input from users who had put login credentials in environment variables. diff --git a/news/3096.bugfix.rst b/news/3096.bugfix.rst new file mode 100644 index 00000000..657abbb2 --- /dev/null +++ b/news/3096.bugfix.rst @@ -0,0 +1 @@ +- Fixed a bug which sometimes prevented cloning and parsing ``mercurial`` requirements. diff --git a/news/3096.feature.rst b/news/3096.feature.rst new file mode 100644 index 00000000..b483d545 --- /dev/null +++ b/news/3096.feature.rst @@ -0,0 +1 @@ +- Added support for python installations managed by ``asdf``. diff --git a/news/3096.vendor.rst b/news/3096.vendor.rst new file mode 100644 index 00000000..ff2cc836 --- /dev/null +++ b/news/3096.vendor.rst @@ -0,0 +1,6 @@ +Updated vendored dependencies: + - ``requests 2.19.1 => 2.20.1`` + - ``tomlkit 0.4.46 => 0.5.2`` + - ``vistir 0.1.6 => 0.2.4`` + - ``pythonfinder 1.1.2 => 1.1.8`` + - ``requirementslib 1.1.10 => 1.3.0`` diff --git a/news/3145.bugfix.rst b/news/3145.bugfix.rst index 61ec9445..16696e27 100644 --- a/news/3145.bugfix.rst +++ b/news/3145.bugfix.rst @@ -1 +1 @@ -Remote non-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. +Hashes for remote andd local non-PyPI artifacts will now be included in ``Pipfile.lock`` during resolution. diff --git a/news/3170.feature b/news/3170.feature index b8bc5218..eae5cbac 100644 --- a/news/3170.feature +++ b/news/3170.feature @@ -1 +1 @@ -Do not show error but success for running pipenv uninstall --all in a fresh virtuanlenv +Do not show error but success for running ``pipenv uninstall --all`` in a fresh virtual environment. diff --git a/news/d65e7c90-3e70-40ba-8242-1e6ed18fc2fe.feature.rst b/news/3217.feature.rst similarity index 100% rename from news/d65e7c90-3e70-40ba-8242-1e6ed18fc2fe.feature.rst rename to news/3217.feature.rst From 15f8d78bceae205fe297eb4ed2772da3daced4de Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 13 Nov 2018 17:46:30 -0500 Subject: [PATCH 179/218] Syntax error and pythonfinder issue fixes Signed-off-by: Dan Ryan --- pipenv/utils.py | 3 +- pipenv/vendor/pythonfinder/models/asdf.py | 9 - pipenv/vendor/pythonfinder/models/path.py | 42 ++-- pipenv/vendor/pythonfinder/models/pyenv.py | 237 --------------------- pipenv/vendor/pythonfinder/utils.py | 10 +- 5 files changed, 29 insertions(+), 272 deletions(-) delete mode 100644 pipenv/vendor/pythonfinder/models/asdf.py delete mode 100644 pipenv/vendor/pythonfinder/models/pyenv.py diff --git a/pipenv/utils.py b/pipenv/utils.py index 265c5797..2f48905a 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1142,8 +1142,7 @@ def normalize_path(path): def get_url_name(url): if not isinstance(url, six.string_types): return - from urllib3.util import parse as urllib3_parse - return urllib3_parse(url).host + return urllib3_util.parse_url(url).host def get_canonical_names(packages): diff --git a/pipenv/vendor/pythonfinder/models/asdf.py b/pipenv/vendor/pythonfinder/models/asdf.py deleted file mode 100644 index 3ba6e4fa..00000000 --- a/pipenv/vendor/pythonfinder/models/asdf.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding=utf-8 -*- -import attr - -from .pyenv import PyenvFinder - - -@attr.s -class AsdfFinder(PyenvFinder): - version_root = attr.ib(default="installs/python/*") diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index d3cdd9d1..53c2360c 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -131,7 +131,8 @@ class SystemPath(object): ) def _get_last_instance(self, path): - paths = [normalize_path(p) for p in reversed(self.path_order)] + reversed_paths = reversed(self.path_order) + paths = [normalize_path(p) for p in reversed_paths] normalized_target = normalize_path(path) last_instance = next( iter(p for p in paths if normalized_target in p), None @@ -150,7 +151,7 @@ class SystemPath(object): ) def _remove_path(self, path): - path_copy = reversed(self.path_order[:]) + path_copy = [p for p in reversed(self.path_order[:])] new_order = [] target = normalize_path(path) path_map = { @@ -163,22 +164,23 @@ class SystemPath(object): normalized = normalize_path(current_path) if normalized != target: new_order.append(normalized) - new_order = reversed(new_order) + new_order = [p for p in reversed(new_order)] self.path_order = new_order def _setup_asdf(self): from .python import PythonFinder + self.asdf_finder = PythonFinder.create( + root=ASDF_DATA_DIR, ignore_unsupported=True, + sort_function=parse_asdf_version_order, version_glob_path="installs/python/*") asdf_index = self._get_last_instance(ASDF_DATA_DIR) if not asdf_index: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here return - self.asdf_finder = PythonFinder.create( - root=ASDF_DATA_DIR, ignore_unsupported=True, - sort_function=parse_asdf_version_order, version_glob_path="installs/python/*") root_paths = [p for p in self.asdf_finder.roots] self._slice_in_paths(asdf_index, root_paths) self.paths.update(self.asdf_finder.roots) + self._remove_path(normalize_path(os.path.join(ASDF_DATA_DIR, "shims"))) self._register_finder("asdf", self.asdf_finder) def _setup_pyenv(self): @@ -452,7 +454,7 @@ class PathEntry(BasePath): only_python = attr.ib(default=False) name = attr.ib() py_version = attr.ib() - pythons = attr.ib() + _pythons = attr.ib(default=attr.Factory(defaultdict)) def __str__(self): return fs_str("{0}".format(self.path.as_posix())) @@ -506,19 +508,19 @@ class PathEntry(BasePath): return py_version return - @pythons.default - def get_pythons(self): - pythons = defaultdict() - if self.is_dir: - for path, entry in self.children.items(): - _path = ensure_path(entry.path) - if entry.is_python: - pythons[_path.as_posix()] = entry - else: - if self.is_python: - _path = ensure_path(self.path) - pythons[_path.as_posix()] = self - return pythons + @property + def pythons(self): + if not self._pythons: + if self.is_dir: + for path, entry in self.children.items(): + _path = ensure_path(entry.path) + if entry.is_python: + self._pythons[_path.as_posix()] = entry + else: + if self.is_python: + _path = ensure_path(self.path) + self._pythons[_path.as_posix()] = self + return self._pythons @cached_property def as_python(self): diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py deleted file mode 100644 index 6f2d6422..00000000 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ /dev/null @@ -1,237 +0,0 @@ -# -*- coding=utf-8 -*- -from __future__ import absolute_import, print_function - -import logging -import operator - -from collections import defaultdict - -import attr -import sysconfig - -from vistir.compat import Path - -from ..utils import ( - ensure_path, - optional_instance_of, - unnest, -) -from .mixins import BaseFinder, BasePath -from .path import SystemPath, PathEntry -from .python import PythonVersion - - -logger = logging.getLogger(__name__) - - -@attr.s(slots=True) -class PyenvFinder(BaseFinder, BasePath): - root = attr.ib(default=None, validator=optional_instance_of(Path)) - #: ignore_unsupported should come before versions, because its value is used - #: in versions's default initializer. - ignore_unsupported = attr.ib(default=True) - paths = attr.ib(default=attr.Factory(list)) - roots = attr.ib(default=attr.Factory(defaultdict)) - version_root = attr.ib(default="versions/*") - versions = attr.ib() - pythons = attr.ib() - - @property - def expanded_paths(self): - return ( - path for path in unnest(p for p in self.versions.values()) - if path is not None - ) - - def get_version_order(self): - version_order_file, version_order_lines = self.root.joinpath("version"), [] - if version_order_file.exists(): - version_order_lines = version_order_file.read_text(encoding="utf-8").splitlines() - - version_paths = [ - p for p in self.root.glob(self.version_root) - if not (p.parent.name == "envs" or p.name == "envs") - ] - versions = {v.name: v for v in version_paths} - version_order = [versions[v] for v in version_order_lines if v in versions] - for version in version_order: - version_paths.remove(version) - version_order += version_paths - return version_order - - @classmethod - def version_from_bin_dir(cls, base_dir, name=None): - py_version = None - version_path = PathEntry.create( - path=base_dir.absolute().as_posix(), - only_python=True, - name=base_dir.parent.name, - ) - py_version = next(iter(version_path.find_all_python_versions()), None) - return py_version - - @versions.default - def get_versions(self): - versions = defaultdict() - bin_ = "{base}/bin" - for p in self.get_version_order(): - bin_dir = Path(bin_.format(base=p.as_posix())) - version_path = None - if bin_dir.exists(): - version_path = PathEntry.create( - path=bin_dir.absolute().as_posix(), - only_python=False, - name=p.name, - is_root=True, - ) - version = None - try: - version = PythonVersion.parse(p.name) - except ValueError: - entry = next(iter(version_path.find_all_python_versions()), None) - if not entry: - if self.ignore_unsupported: - continue - raise - else: - version = entry.py_version.as_dict() - except Exception: - if not self.ignore_unsupported: - raise - logger.warning( - "Unsupported Python version %r, ignoring...", p.name, exc_info=True - ) - continue - if not version: - continue - version_tuple = ( - version.get("major"), - version.get("minor"), - version.get("patch"), - version.get("is_prerelease"), - version.get("is_devrelease"), - version.get("is_debug"), - ) - self.roots[p] = version_path - versions[version_tuple] = version_path - self.paths.append(version_path) - return versions - - @pythons.default - def get_pythons(self): - pythons = defaultdict() - for p in self.paths: - pythons.update(p.pythons) - return pythons - - @classmethod - def create(cls, root, ignore_unsupported=True): - root = ensure_path(root) - return cls(root=root, ignore_unsupported=ignore_unsupported) - - def find_all_python_versions( - self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, - ): - """Search for a specific python version on the path. Return all copies - - :param major: Major python version to search for. - :type major: int - :param int minor: Minor python version to search for, defaults to None - :param int patch: Patch python version to search for, defaults to None - :param bool pre: Search for prereleases (default None) - prioritize releases if None - :param bool dev: Search for devreleases (default None) - prioritize releases if None - :param str arch: Architecture to include, e.g. '64bit', defaults to None - :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` - :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. - :rtype: List[:class:`~pythonfinder.models.PathEntry`] - """ - - version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, - ) - py = operator.attrgetter("as_python") - pythons = ( - py_ver for py_ver in (py(p) for p in self.pythons.values() if p is not None) - if py_ver is not None - ) - # pythons = filter(None, [p.as_python for p in self.pythons.values()]) - matching_versions = filter(lambda py: version_matcher(py), pythons) - version_sort = operator.attrgetter("version_sort") - return sorted(matching_versions, key=version_sort, reverse=True) - - def find_python_version( - self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, - ): - """Search or self for the specified Python version and return the first match. - - :param major: Major version number. - :type major: int - :param int minor: Minor python version to search for, defaults to None - :param int patch: Patch python version to search for, defaults to None - :param bool pre: Search for prereleases (default None) - prioritize releases if None - :param bool dev: Search for devreleases (default None) - prioritize releases if None - :param str arch: Architecture to include, e.g. '64bit', defaults to None - :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` - :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. - """ - - version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, - ) - pythons = filter(None, [p.as_python for p in self.pythons.values()]) - matching_versions = filter(lambda py: version_matcher(py), pythons) - version_sort = operator.attrgetter("version_sort") - return next(iter(c for c in sorted(matching_versions, key=version_sort, reverse=True)), None) - - -@attr.s -class VersionPath(SystemPath): - base = attr.ib(default=None, validator=optional_instance_of(Path)) - name = attr.ib(default=None) - - @classmethod - def create(cls, path, only_python=True, pythons=None, name=None): - """Accepts a path to a base python version directory. - - Generates the pyenv version listings for it""" - path = ensure_path(path) - path_entries = defaultdict(PathEntry) - bin_ = sysconfig._INSTALL_SCHEMES[sysconfig._get_default_scheme()]["scripts"] - if path.as_posix().endswith(Path(bin_).name): - path = path.parent - bin_dir = ensure_path(bin_.format(base=path.as_posix())) - if not name: - name = path.name - current_entry = PathEntry.create( - bin_dir, is_root=True, only_python=True, pythons=pythons, name=name - ) - path_entries[bin_dir.as_posix()] = current_entry - return cls(name=name, base=bin_dir, paths=path_entries) diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 619fadef..a26d0548 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -12,7 +12,7 @@ import six import vistir -from .environment import PYENV_INSTALLED, PYENV_ROOT, ASDF_INSTALLED, ASDF_DATA_DIR +from .environment import PYENV_ROOT from .exceptions import InvalidPythonVersion try: @@ -90,7 +90,7 @@ def looks_like_python(name): @lru_cache(maxsize=1024) def path_is_python(path): - return path_is_known_executable(path) and looks_like_python(path.name) + return path_is_executable(path) and looks_like_python(path.name) @lru_cache(maxsize=1024) @@ -116,7 +116,9 @@ def _filter_none(k, v): def normalize_path(path): - return os.path.normpath(os.path.normcase(os.path.abspath(str(path)))) + return os.path.normpath(os.path.normcase( + os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) + )) @lru_cache(maxsize=1024) @@ -161,7 +163,7 @@ def parse_asdf_version_order(filename=".tool-versions"): line for line in contents.splitlines() if line.startswith("python") ), None) if python_section: - python_key, versions = python_section.partition() + python_key, _, versions = python_section.partition(" ") if versions: return versions.split() From f494571183eaa55da58e27e5881b8299f37e9ccc Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 13 Nov 2018 18:07:48 -0500 Subject: [PATCH 180/218] Code cleanup -- unused functions Signed-off-by: Dan Ryan --- pipenv/_compat.py | 31 ++++----- pipenv/utils.py | 158 ---------------------------------------------- 2 files changed, 13 insertions(+), 176 deletions(-) diff --git a/pipenv/_compat.py b/pipenv/_compat.py index fb2c0147..1dc52d5f 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -4,6 +4,7 @@ Exposes a standard API that enables compatibility across python versions, operating systems, etc. """ + import functools import importlib import io @@ -66,21 +67,10 @@ from vistir.compat import ResourceWarning warnings.filterwarnings("ignore", category=ResourceWarning) -def pip_import(module_path, subimport=None, old_path=None): - internal = "pip._internal.{0}".format(module_path) - old_path = old_path or module_path - pip9 = "pip.{0}".format(old_path) - try: - _tmp = importlib.import_module(internal) - except ImportError: - _tmp = importlib.import_module(pip9) - if subimport: - return getattr(_tmp, subimport, _tmp) - return _tmp - - class TemporaryDirectory(object): - """Create and return a temporary directory. This has the same + + """ + Create and return a temporary directory. This has the same behavior as mkdtemp but can be used as a context manager. For example: @@ -146,9 +136,11 @@ def _sanitize_params(prefix, suffix, dir): class _TemporaryFileCloser: - """A separate object allowing proper closing of a temporary file's + """ + A separate object allowing proper closing of a temporary file's underlying file object, without adding a __del__ method to the - temporary file.""" + temporary file. + """ file = None # Set here since __del__ checks it close_called = False @@ -192,7 +184,9 @@ class _TemporaryFileCloser: class _TemporaryFileWrapper: - """Temporary file wrapper + + """ + Temporary file wrapper This class provides a wrapper around files opened for temporary use. In particular, it seeks to automatically remove the file when it is no longer needed. @@ -268,7 +262,8 @@ def NamedTemporaryFile( dir=None, delete=True, ): - """Create and return a temporary file. + """ + Create and return a temporary file. Arguments: 'prefix', 'suffix', 'dir' -- as for mkstemp. 'mode' -- the mode argument to io.open (default "w+b"). diff --git a/pipenv/utils.py b/pipenv/utils.py index 2f48905a..9f62e2b2 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -778,13 +778,6 @@ def resolve_deps( return results -def multi_split(s, split): - """Splits on multiple given separators.""" - for r in split: - s = s.replace(r, "|") - return [i for i in s.split("|") if len(i) > 0] - - def is_star(val): return isinstance(val, six.string_types) and val == "*" @@ -855,22 +848,6 @@ def is_required_version(version, specified_version): return True -def strip_ssh_from_git_uri(uri): - """Return git+ssh:// formatted URI to git+git@ format""" - if isinstance(uri, six.string_types): - uri = uri.replace("git+ssh://", "git+") - return uri - - -def clean_git_uri(uri): - """Cleans VCS uris from pip format""" - if isinstance(uri, six.string_types): - # Add scheme for parsing purposes, this is also what pip does - if uri.startswith("git+") and "://" not in uri: - uri = uri.replace("git+", "git+ssh://") - return uri - - def is_editable(pipfile_entry): if hasattr(pipfile_entry, "get"): return pipfile_entry.get("editable", False) and any( @@ -1013,94 +990,6 @@ def split_section(input_file, section_suffix, test_function): return input_file -def split_file(file_dict): - """Split VCS and editable dependencies out from file.""" - from .vendor.requirementslib.utils import is_vcs - sections = { - "vcs": is_vcs, - "editable": lambda x: hasattr(x, "keys") and x.get("editable"), - } - for k, func in sections.items(): - file_dict = split_section(file_dict, k, func) - return file_dict - - -def merge_deps( - file_dict, - project, - dev=False, - requirements=False, - ignore_hashes=False, - blocking=False, - only=False, -): - """ - Given a file_dict, merges dependencies and converts them to pip dependency lists. - :param dict file_dict: The result of calling :func:`pipenv.utils.split_file` - :param :class:`pipenv.project.Project` project: Pipenv project - :param bool dev=False: Flag indicating whether dev dependencies are to be installed - :param bool requirements=False: Flag indicating whether to use a requirements file - :param bool ignore_hashes=False: - :param bool blocking=False: - :param bool only=False: - :return: Pip-converted 3-tuples of [deps, requirements_deps] - """ - deps = [] - requirements_deps = [] - for section in list(file_dict.keys()): - # Turn develop-vcs into ['develop', 'vcs'] - section_name, suffix = ( - section.rsplit("-", 1) - if "-" in section and not section == "dev-packages" - else (section, None) - ) - if not file_dict[section] or section_name not in ( - "dev-packages", - "packages", - "default", - "develop", - ): - continue - - is_dev = section_name in ("dev-packages", "develop") - if is_dev and not dev: - continue - - if ignore_hashes: - for k, v in file_dict[section]: - if "hash" in v: - del v["hash"] - # Block and ignore hashes for all suffixed sections (vcs/editable) - no_hashes = True if suffix else ignore_hashes - block = True if suffix else blocking - include_index = True if not suffix else False - converted = convert_deps_to_pip( - file_dict[section], project, r=False, include_index=include_index - ) - deps.extend((d, no_hashes, block) for d in converted) - if dev and is_dev and requirements: - requirements_deps.extend((d, no_hashes, block) for d in converted) - return deps, requirements_deps - - -def recase_file(file_dict): - """Recase file before writing to output.""" - if "packages" in file_dict or "dev-packages" in file_dict: - sections = ("packages", "dev-packages") - elif "default" in file_dict or "develop" in file_dict: - sections = ("default", "develop") - for section in sections: - file_section = file_dict.get(section, {}) - # Try to properly case each key if we can. - for key in list(file_section.keys()): - try: - cased_key = proper_case(key) - except IOError: - cased_key = key - file_section[cased_key] = file_section.pop(key) - return file_dict - - def get_windows_path(*args): """Sanitize a path for windows environments @@ -1355,13 +1244,6 @@ def safe_expandvars(value): return value -def extract_uri_from_vcs_dep(dep): - valid_keys = VCS_LIST + ("uri", "file") - if hasattr(dep, "keys"): - return first(dep[k] for k in valid_keys if k in dep) or None - return None - - def get_vcs_deps( project, which=None, @@ -1570,46 +1452,6 @@ def parse_indexes(line): return indexes, trusted_hosts, remainder -def fix_venv_site(venv_lib_dir): - # From https://github.com/pypa/pip/blob/master/tests/lib/venv.py#L84 - # Prevent accidental inclusions of site packages during virtualenv operations - from .vendor.vistir.compat import Path - import compileall - site_py = Path(venv_lib_dir).joinpath('site.py').as_posix() - with open(site_py) as fp: - site_contents = fp.read() - for pattern, replace in ( - ( - # Ensure enabling user site does not result in adding - # the real site-packages' directory to `sys.path`. - ( - '\ndef virtual_addsitepackages(known_paths):\n' - ), - ( - '\ndef virtual_addsitepackages(known_paths):\n' - ' return known_paths\n' - ), - ), - ( - # Fix sites ordering: user site must be added before system. - ( - '\n paths_in_sys = addsitepackages(paths_in_sys)' - '\n paths_in_sys = addusersitepackages(paths_in_sys)\n' - ), - ( - '\n paths_in_sys = addusersitepackages(paths_in_sys)' - '\n paths_in_sys = addsitepackages(paths_in_sys)\n' - ), - ), - ): - if pattern in site_contents and replace not in site_contents: - site_contents = site_contents.replace(pattern, replace) - with open(site_py, 'w') as fp: - fp.write(site_contents) - # Make sure bytecode is up-to-date too. - assert compileall.compile_file(str(site_py), quiet=1, force=True) - - @contextmanager def sys_version(version_tuple): """ From 2e10ff6c7d3735bd98c806b94e7df1b5640f6fee Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 13 Nov 2018 18:08:00 -0500 Subject: [PATCH 181/218] Fix pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 53c2360c..7fae3312 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -33,6 +33,11 @@ from ..utils import ( from .python import PythonVersion +ASDF_SHIM_PATH = normalize_path(os.path.join(ASDF_DATA_DIR, "shims")) +PYENV_SHIM_PATH = normalize_path(os.path.join(PYENV_ROOT, "shims")) +SHIM_PATHS = [ASDF_SHIM_PATH, PYENV_SHIM_PATH] + + @attr.s class SystemPath(object): global_search = attr.ib(default=True) @@ -434,6 +439,7 @@ class SystemPath(object): path=p.absolute(), is_root=True, only_python=only_python ) for p in _path_objects + if not any(shim in normalize_path(str(p)) for shim in SHIM_PATHS) } ) return cls( @@ -476,6 +482,8 @@ class PathEntry(BasePath): yield (self.path.as_posix(), copy.deepcopy(self)) elif self.is_root: for child in self._filter_children(): + if any(shim in normalize_path(str(child)) for shim in SHIM_PATHS): + continue yield (child.as_posix(), PathEntry.create(path=child, **pass_args)) return @@ -566,6 +574,8 @@ class PathEntry(BasePath): if not guessed_name: child_creation_args["name"] = name for pth, python in pythons.items(): + if any(shim in normalize_path(str(pth)) for shim in SHIM_PATHS): + continue pth = ensure_path(pth) children[pth.as_posix()] = PathEntry( py_version=python, From f9b97dacc7afa3d1c3592a6222fd4b2ca32b8cec Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 13 Nov 2018 19:07:06 -0500 Subject: [PATCH 182/218] Get rid of split file test Signed-off-by: Dan Ryan --- tests/unit/test_utils.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 40977ede..422c1002 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -169,32 +169,6 @@ class TestUtils: from pipenv.vendor.requirementslib.utils import is_vcs assert is_vcs(entry) is expected - @pytest.mark.utils - def test_split_file(self): - pipfile_dict = { - "packages": { - "requests": {"git": "https://github.com/kennethreitz/requests.git"}, - "Flask": "*", - "tablib": {"path": ".", "editable": True}, - }, - "dev-packages": { - "Django": "==1.10", - "click": {"svn": "https://svn.notareal.com/click"}, - "crayons": {"hg": "https://hg.alsonotreal.com/crayons"}, - }, - } - split_dict = pipenv.utils.split_file(pipfile_dict) - assert list(split_dict["packages"].keys()) == ["Flask"] - assert split_dict["packages-vcs"] == { - "requests": {"git": "https://github.com/kennethreitz/requests.git"} - } - assert split_dict["packages-editable"] == { - "tablib": {"path": ".", "editable": True} - } - assert list(split_dict["dev-packages"].keys()) == ["Django"] - assert "click" in split_dict["dev-packages-vcs"] - assert "crayons" in split_dict["dev-packages-vcs"] - @pytest.mark.utils def test_python_version_from_bad_path(self): assert pipenv.utils.python_version("/fake/path") is None From 095c9ef73000cd9616f1faff0d0dfc017628caa4 Mon Sep 17 00:00:00 2001 From: frostming Date: Wed, 14 Nov 2018 13:12:22 +0800 Subject: [PATCH 183/218] remove useless tests --- news/3145.bugfix.rst | 2 +- tests/unit/test_utils.py | 26 -------------------------- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/news/3145.bugfix.rst b/news/3145.bugfix.rst index 16696e27..e0ed5095 100644 --- a/news/3145.bugfix.rst +++ b/news/3145.bugfix.rst @@ -1 +1 @@ -Hashes for remote andd local non-PyPI artifacts will now be included in ``Pipfile.lock`` during resolution. +Hashes for remote and local non-PyPI artifacts will now be included in ``Pipfile.lock`` during resolution. diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 40977ede..422c1002 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -169,32 +169,6 @@ class TestUtils: from pipenv.vendor.requirementslib.utils import is_vcs assert is_vcs(entry) is expected - @pytest.mark.utils - def test_split_file(self): - pipfile_dict = { - "packages": { - "requests": {"git": "https://github.com/kennethreitz/requests.git"}, - "Flask": "*", - "tablib": {"path": ".", "editable": True}, - }, - "dev-packages": { - "Django": "==1.10", - "click": {"svn": "https://svn.notareal.com/click"}, - "crayons": {"hg": "https://hg.alsonotreal.com/crayons"}, - }, - } - split_dict = pipenv.utils.split_file(pipfile_dict) - assert list(split_dict["packages"].keys()) == ["Flask"] - assert split_dict["packages-vcs"] == { - "requests": {"git": "https://github.com/kennethreitz/requests.git"} - } - assert split_dict["packages-editable"] == { - "tablib": {"path": ".", "editable": True} - } - assert list(split_dict["dev-packages"].keys()) == ["Django"] - assert "click" in split_dict["dev-packages-vcs"] - assert "crayons" in split_dict["dev-packages-vcs"] - @pytest.mark.utils def test_python_version_from_bad_path(self): assert pipenv.utils.python_version("/fake/path") is None From 08c384bba0f9f1c051ba3057b92e12f0cb42bb17 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 00:28:32 -0500 Subject: [PATCH 184/218] Implement `auto_envvar_prefix` - Closes #2200 Signed-off-by: Dan Ryan --- news/2200.feature.rst | 1 + pipenv/__main__.py | 2 +- pipenv/cli/options.py | 39 ++++++++++++++++++++------------------- 3 files changed, 22 insertions(+), 20 deletions(-) create mode 100644 news/2200.feature.rst diff --git a/news/2200.feature.rst b/news/2200.feature.rst new file mode 100644 index 00000000..daa4b215 --- /dev/null +++ b/news/2200.feature.rst @@ -0,0 +1 @@ +Added persistent settings for all CLI flags via ``PIPENV_{FLAG_NAME}`` environment variables by enabling ``auto_envvar_prefix=PIPENV`` in click (implements PEEP-0002). diff --git a/pipenv/__main__.py b/pipenv/__main__.py index 98dcca0c..56494106 100644 --- a/pipenv/__main__.py +++ b/pipenv/__main__.py @@ -1,4 +1,4 @@ from .cli import cli if __name__ == "__main__": - cli() + cli(auto_envvar_prefix="PIPENV") diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 208c0c66..da06604a 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -106,8 +106,8 @@ def editable_option(f): state.installstate.editables.extend(value) return value return option('-e', '--editable', expose_value=False, multiple=True, - help='An editable python package URL or path, often to a VCS repo.', - callback=callback)(f) + help='An editable python package URL or path, often to a VCS repo.', + callback=callback, type=click.types.STRING)(f) def sequential_option(f): @@ -157,7 +157,7 @@ def ignore_pipfile_option(f): return value return option("--ignore-pipfile", is_flag=True, default=False, expose_value=False, help="Ignore Pipfile when installing, using the Pipfile.lock.", - callback=callback)(f) + callback=callback, type=click.types.BOOL)(f) def dev_option(f): @@ -184,7 +184,8 @@ def package_arg(f): state = ctx.ensure_object(State) state.installstate.packages.extend(value) return value - return argument('packages', nargs=-1, callback=callback, expose_value=False,)(f) + return argument('packages', nargs=-1, callback=callback, expose_value=False, + type=click.types.STRING)(f) def three_option(f): @@ -195,8 +196,8 @@ def three_option(f): state.two = not value return value return option("--three/--two", is_flag=True, default=None, - help="Use Python 3/2 when creating virtualenv.", callback=callback, - expose_value=False)(f) + help="Use Python 3/2 when creating virtualenv.", callback=callback, + expose_value=False)(f) def python_option(f): @@ -206,8 +207,8 @@ def python_option(f): state.python = validate_python_path(ctx, param, value) return value return option("--python", default=False, nargs=1, callback=callback, - help="Specify which version of Python virtualenv should use.", - expose_value=False)(f) + help="Specify which version of Python virtualenv should use.", + expose_value=False)(f) def pypi_mirror_option(f): @@ -217,7 +218,7 @@ def pypi_mirror_option(f): state.pypi_mirror = validate_pypi_mirror(ctx, param, value) return value return option("--pypi-mirror", default=environments.PIPENV_PYPI_MIRROR, nargs=1, - callback=callback, help="Specify a PyPI mirror.", expose_value=False)(f) + callback=callback, help="Specify a PyPI mirror.", expose_value=False)(f) def verbose_option(f): @@ -227,7 +228,7 @@ def verbose_option(f): state.verbose = True setup_verbosity(ctx, param, value) return option("--verbose", "-v", is_flag=True, expose_value=False, - callback=callback, help="Verbose mode.")(f) + callback=callback, help="Verbose mode.", type=click.types.BOOL)(f) def site_packages_option(f): @@ -236,8 +237,8 @@ def site_packages_option(f): state.site_packages = value return value return option("--site-packages", is_flag=True, default=False, type=click.types.BOOL, - help="Enable site-packages for the virtualenv.", callback=callback, - expose_value=False)(f) + help="Enable site-packages for the virtualenv.", callback=callback, + expose_value=False)(f) def clear_option(f): @@ -246,8 +247,8 @@ def clear_option(f): state.clear = value return value return option("--clear", is_flag=True, callback=callback, type=click.types.BOOL, - help="Clears caches (pipenv, pip, and pip-tools).", - expose_value=False)(f) + help="Clears caches (pipenv, pip, and pip-tools).", + expose_value=False)(f) def system_option(f): @@ -257,7 +258,7 @@ def system_option(f): state.system = value return value return option("--system", is_flag=True, default=False, help="System pip management.", - callback=callback, type=click.types.BOOL, expose_value=False)(f) + callback=callback, type=click.types.BOOL, expose_value=False)(f) def requirementstxt_option(f): @@ -267,7 +268,7 @@ def requirementstxt_option(f): state.installstate.requirementstxt = value return value return option("--requirements", "-r", nargs=1, default=False, expose_value=False, - help="Import a requirements.txt file.", callback=callback)(f) + help="Import a requirements.txt file.", callback=callback)(f) def requirements_flag(f): @@ -277,7 +278,7 @@ def requirements_flag(f): state.installstate.requirementstxt = value return value return option("--requirements", "-r", default=False, is_flag=True, expose_value=False, - help="Generate output in requirements.txt format.", callback=callback)(f) + help="Generate output in requirements.txt format.", callback=callback)(f) def code_option(f): @@ -296,8 +297,8 @@ def deploy_option(f): state.installstate.deploy = value return value return option("--deploy", is_flag=True, default=False, type=click.types.BOOL, - help=u"Abort if the Pipfile.lock is out-of-date, or Python version is" - " wrong.", callback=callback, expose_value=False)(f) + help=u"Abort if the Pipfile.lock is out-of-date, or Python version is" + " wrong.", callback=callback, expose_value=False)(f) def setup_verbosity(ctx, param, value): From 2f97279fa8db1b177db60e3ccab3ba73cedb46a0 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 00:29:04 -0500 Subject: [PATCH 185/218] Update vendor file Signed-off-by: Dan Ryan --- pipenv/vendor/vendor.txt | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index c106a59c..f82df6a7 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -3,10 +3,10 @@ backports.shutil_get_terminal_size==1.0.0 backports.weakref==1.0.post1 blindspin==2.0.1 click==7.0 -click-completion==0.4.1 +click-completion==0.5.0 click-didyoumean==0.0.3 colorama==0.3.9 -delegator.py==0.1.1 +delegator.py==0.1.2 pexpect==4.6.0 ptyprocess==0.6.0 python-dotenv==0.9.1 @@ -21,18 +21,17 @@ pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.8 -requests==2.20.0 +pythonfinder==1.1.9 +requests==2.20.1 chardet==3.0.4 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.3.0 +requirementslib==1.3.1 attrs==18.2.0 distlib==0.2.8 - packaging==18.0 + packaging==18.1 pyparsing==2.2.2 - pytoml==0.1.19 plette==0.2.2 tomlkit==0.5.2 shellingham==1.2.7 @@ -41,7 +40,7 @@ semver==2.8.1 shutilwhich==1.1.0 toml==0.10.0 cached-property==1.4.3 -vistir==0.2.2 +vistir==0.2.4 pip-shims==0.3.2 ptyprocess==0.6.0 enum34==1.1.6 From ec166f9295ee6892509fa3fcc40e09c8ea4ae58c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 00:43:11 -0500 Subject: [PATCH 186/218] Reformat cli and command code Signed-off-by: Dan Ryan --- pipenv/cli/command.py | 2 +- pipenv/cli/options.py | 36 ++++++++++++++++++------------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 74c57720..1480a0e7 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -226,6 +226,7 @@ def install( ): """Installs provided packages and adds them to Pipfile, or (if no packages are given), installs all packages from Pipfile.""" from ..core import do_install + echo("Skip lock value: %s" % state.installstate.skip_lock) retcode = do_install( dev=state.installstate.dev, @@ -279,7 +280,6 @@ def uninstall( ): """Un-installs a provided package and removes it from Pipfile.""" from ..core import do_uninstall - retcode = do_uninstall( packages=state.installstate.packages, editable_packages=state.installstate.editables, diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index da06604a..2242b4e3 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -86,8 +86,8 @@ def index_option(f): state.index = value return value return option('-i', '--index', expose_value=False, envvar="PIP_INDEX_URL", - help='Target PyPI-compatible package index url.', nargs=1, - callback=callback)(f) + help='Target PyPI-compatible package index url.', nargs=1, + callback=callback)(f) def extra_index_option(f): @@ -96,8 +96,8 @@ def extra_index_option(f): state.extra_index_urls.extend(list(value)) return value return option("--extra-index-url", multiple=True, expose_value=False, - help=u"URLs to the extra PyPI compatible indexes to query for package lookups.", - callback=callback, envvar="PIP_EXTRA_INDEX_URL")(f) + help=u"URLs to the extra PyPI compatible indexes to query for package lookups.", + callback=callback, envvar="PIP_EXTRA_INDEX_URL")(f) def editable_option(f): @@ -116,8 +116,8 @@ def sequential_option(f): state.installstate.sequential = value return value return option("--sequential", is_flag=True, default=False, expose_value=False, - help="Install dependencies one-at-a-time, instead of concurrently.", - callback=callback, type=click.types.BOOL)(f) + help="Install dependencies one-at-a-time, instead of concurrently.", + callback=callback, type=click.types.BOOL)(f) def skip_lock_option(f): @@ -126,8 +126,8 @@ def skip_lock_option(f): state.installstate.skip_lock = value return value return option("--skip-lock", is_flag=True, default=False, expose_value=False, - help=u"Skip locking mechanisms and use the Pipfile instead during operation.", - callback=callback, type=click.types.BOOL)(f) + help=u"Skip locking mechanisms and use the Pipfile instead during operation.", + callback=callback, type=click.types.BOOL)(f) def keep_outdated_option(f): @@ -136,8 +136,8 @@ def keep_outdated_option(f): state.installstate.keep_outdated = value return value return option("--keep-outdated", is_flag=True, default=False, expose_value=False, - help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", - callback=callback, type=click.types.BOOL)(f) + help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", + callback=callback, type=click.types.BOOL)(f) def selective_upgrade_option(f): @@ -146,8 +146,8 @@ def selective_upgrade_option(f): state.installstate.selective_upgrade = value return value return option("--selective-upgrade", is_flag=True, default=False, type=click.types.BOOL, - help="Update specified packages.", callback=callback, - expose_value=False)(f) + help="Update specified packages.", callback=callback, + expose_value=False)(f) def ignore_pipfile_option(f): @@ -156,8 +156,8 @@ def ignore_pipfile_option(f): state.installstate.ignore_pipfile = value return value return option("--ignore-pipfile", is_flag=True, default=False, expose_value=False, - help="Ignore Pipfile when installing, using the Pipfile.lock.", - callback=callback, type=click.types.BOOL)(f) + help="Ignore Pipfile when installing, using the Pipfile.lock.", + callback=callback, type=click.types.BOOL)(f) def dev_option(f): @@ -166,8 +166,8 @@ def dev_option(f): state.installstate.dev = value return value return option("--dev", "-d", is_flag=True, default=False, type=click.types.BOOL, - help="Install both develop and default packages.", callback=callback, - expose_value=False)(f) + help="Install both develop and default packages.", callback=callback, + expose_value=False)(f) def pre_option(f): @@ -176,7 +176,7 @@ def pre_option(f): state.installstate.pre = value return value return option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.", - callback=callback, type=click.types.BOOL, expose_value=False)(f) + callback=callback, type=click.types.BOOL, expose_value=False)(f) def package_arg(f): @@ -298,7 +298,7 @@ def deploy_option(f): return value return option("--deploy", is_flag=True, default=False, type=click.types.BOOL, help=u"Abort if the Pipfile.lock is out-of-date, or Python version is" - " wrong.", callback=callback, expose_value=False)(f) + " wrong.", callback=callback, expose_value=False)(f) def setup_verbosity(ctx, param, value): From 86c894d81b2378ca8d962b1af3b904f81265d4d8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 00:45:47 -0500 Subject: [PATCH 187/218] Woops, delegator isn't updated, neither packaging Signed-off-by: Dan Ryan --- pipenv/vendor/vendor.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index f82df6a7..0a98277b 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -6,7 +6,7 @@ click==7.0 click-completion==0.5.0 click-didyoumean==0.0.3 colorama==0.3.9 -delegator.py==0.1.2 +delegator.py==0.1.1 pexpect==4.6.0 ptyprocess==0.6.0 python-dotenv==0.9.1 @@ -30,7 +30,7 @@ requests==2.20.1 requirementslib==1.3.1 attrs==18.2.0 distlib==0.2.8 - packaging==18.1 + packaging==18.0 pyparsing==2.2.2 plette==0.2.2 tomlkit==0.5.2 From ada66d3e72d53ce0d2051b626a85dc0a593eacc5 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 00:51:26 -0500 Subject: [PATCH 188/218] Remove old patch Signed-off-by: Dan Ryan --- .../patches/vendor/vistir-spin-colorama.patch | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 tasks/vendoring/patches/vendor/vistir-spin-colorama.patch diff --git a/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch b/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch deleted file mode 100644 index 69e4cac7..00000000 --- a/tasks/vendoring/patches/vendor/vistir-spin-colorama.patch +++ /dev/null @@ -1,28 +0,0 @@ -diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py -index 2a848922..57a90277 100644 ---- a/pipenv/vendor/vistir/spin.py -+++ b/pipenv/vendor/vistir/spin.py -@@ -5,6 +5,7 @@ import os - import signal - import sys - -+import colorama - import cursor - import six - -@@ -31,6 +32,7 @@ CLEAR_LINE = chr(27) + "[K" - - class DummySpinner(object): - def __init__(self, text="", **kwargs): -+ colorama.init() - self.text = to_native_string(text) - self.stdout = kwargs.get("stdout", sys.stdout) - self.stderr = kwargs.get("stderr", sys.stderr) -@@ -112,7 +114,6 @@ class VistirSpinner(base_obj): - """ - - self.handler = handler -- import colorama - colorama.init() - sigmap = {} - if handler: From bfa0b290cca65f3ed8b95350aaf37eb1ed794d7b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 01:11:59 -0500 Subject: [PATCH 189/218] Update vendored deps Signed-off-by: Dan Ryan --- pipenv/vendor/click_completion/__init__.py | 2 +- pipenv/vendor/click_completion/lib.py | 11 +- pipenv/vendor/passa/cli/options.py | 7 +- pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/models/python.py | 4 +- pipenv/vendor/pythonfinder/pythonfinder.py | 3 +- pipenv/vendor/pythonfinder/utils.py | 22 +- pipenv/vendor/pytoml/LICENSE | 16 - pipenv/vendor/pytoml/__init__.py | 3 - pipenv/vendor/pytoml/core.py | 13 - pipenv/vendor/pytoml/parser.py | 374 ------------------ pipenv/vendor/pytoml/writer.py | 127 ------ pipenv/vendor/requests/__version__.py | 4 +- pipenv/vendor/requests/sessions.py | 13 +- pipenv/vendor/requests/utils.py | 4 +- pipenv/vendor/requirementslib/__init__.py | 2 +- pipenv/vendor/requirementslib/exceptions.py | 4 +- .../vendor/requirementslib/models/pipfile.py | 20 +- .../requirementslib/models/setup_info.py | 3 +- pipenv/vendor/requirementslib/utils.py | 2 +- pipenv/vendor/vistir/__init__.py | 2 +- pipenv/vendor/vistir/contextmanagers.py | 2 +- pipenv/vendor/vistir/path.py | 9 +- 23 files changed, 70 insertions(+), 579 deletions(-) delete mode 100644 pipenv/vendor/pytoml/LICENSE delete mode 100644 pipenv/vendor/pytoml/__init__.py delete mode 100644 pipenv/vendor/pytoml/core.py delete mode 100644 pipenv/vendor/pytoml/parser.py delete mode 100644 pipenv/vendor/pytoml/writer.py diff --git a/pipenv/vendor/click_completion/__init__.py b/pipenv/vendor/click_completion/__init__.py index b849ae23..e30cc0e3 100644 --- a/pipenv/vendor/click_completion/__init__.py +++ b/pipenv/vendor/click_completion/__init__.py @@ -19,7 +19,7 @@ from click_completion.core import completion_configuration, get_code, install, s from click_completion.lib import get_auto_shell from click_completion.patch import patch as _patch -__version__ = '0.4.1' +__version__ = '0.5.0' _initialized = False diff --git a/pipenv/vendor/click_completion/lib.py b/pipenv/vendor/click_completion/lib.py index cd53bc03..167ecfd8 100644 --- a/pipenv/vendor/click_completion/lib.py +++ b/pipenv/vendor/click_completion/lib.py @@ -59,7 +59,7 @@ def double_quote(s): return '"' + s.replace('"', '"\'"\'"') + '"' -def resolve_ctx(cli, prog_name, args): +def resolve_ctx(cli, prog_name, args, resilient_parsing=True): """ Parameters @@ -76,13 +76,18 @@ def resolve_ctx(cli, prog_name, args): click.core.Context A new context corresponding to the current command """ - ctx = cli.make_context(prog_name, list(args), resilient_parsing=True) + ctx = cli.make_context(prog_name, list(args), resilient_parsing=resilient_parsing) while ctx.args + ctx.protected_args and isinstance(ctx.command, MultiCommand): a = ctx.protected_args + ctx.args cmd = ctx.command.get_command(ctx, a[0]) if cmd is None: return None - ctx = cmd.make_context(a[0], a[1:], parent=ctx, resilient_parsing=True) + if hasattr(cmd, "no_args_is_help"): + no_args_is_help = cmd.no_args_is_help + cmd.no_args_is_help = False + ctx = cmd.make_context(a[0], a[1:], parent=ctx, resilient_parsing=resilient_parsing) + if hasattr(cmd, "no_args_is_help"): + cmd.no_args_is_help = no_args_is_help return ctx diff --git a/pipenv/vendor/passa/cli/options.py b/pipenv/vendor/passa/cli/options.py index da89a3b1..f8ba1fe7 100644 --- a/pipenv/vendor/passa/cli/options.py +++ b/pipenv/vendor/passa/cli/options.py @@ -46,9 +46,12 @@ class Option(object): class ArgumentGroup(object): - def __init__(self, name, parser=None, is_mutually_exclusive=False, required=None, options=[]): + def __init__( + self, name, parser=None, + is_mutually_exclusive=False, + required=None, options=None): self.name = name - self.options = options + self.options = options or [] self.parser = parser self.required = required self.is_mutually_exclusive = is_mutually_exclusive diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 85666b5c..fd9ec1be 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.8' +__version__ = '1.1.9' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 7feee84e..db7d011e 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -414,8 +414,8 @@ class PythonVersion(object): if not path.is_python and not IGNORE_UNSUPPORTED: raise ValueError("Not a valid python path: %s" % path.path) return - py_version = get_python_version(path.path.as_posix()) - instance_dict = cls.parse(py_version) + py_version = get_python_version(path.path.absolute().as_posix()) + instance_dict = cls.parse(py_version.strip()) if not isinstance(instance_dict.get("version"), Version) and not IGNORE_UNSUPPORTED: raise ValueError("Not a valid python path: %s" % path.path) return diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index b3bad570..011754ea 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -9,7 +9,8 @@ from vistir.compat import lru_cache class Finder(object): def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=True): - """Finder A cross-platform Finder for locating python and other executables. + """ + Finder A cross-platform Finder for locating python and other executables. Searches for python and other specified binaries starting in `path`, if supplied, but searching the bin path of `sys.executable` if `system=True`, and then diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index a26d0548..2debd80e 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -12,17 +12,17 @@ import six import vistir -from .environment import PYENV_ROOT +from .environment import PYENV_ROOT, ASDF_DATA_DIR from .exceptions import InvalidPythonVersion +six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) +from six.moves import Iterable + try: from functools import lru_cache except ImportError: from backports.functools_lru_cache import lru_cache -six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) -from six.moves import Iterable - PYTHON_IMPLEMENTATIONS = ( "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", @@ -52,7 +52,7 @@ def get_python_version(path): version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, - combine_stderr=False) + combine_stderr=False) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) if not c.out: @@ -64,10 +64,6 @@ def optional_instance_of(cls): return attr.validators.optional(attr.validators.instance_of(cls)) -def path_and_exists(path): - return attr.validators.instance_of(vistir.compat.Path) and path.exists() - - def path_is_executable(path): return os.access(str(path), os.X_OK) @@ -95,7 +91,8 @@ def path_is_python(path): @lru_cache(maxsize=1024) def ensure_path(path): - """Given a path (either a string or a Path object), expand variables and return a Path object. + """ + Given a path (either a string or a Path object), expand variables and return a Path object. :param path: A string or a :class:`~pathlib.Path` object. :type path: str or :class:`~pathlib.Path` @@ -115,6 +112,7 @@ def _filter_none(k, v): return False +# TODO: Reimplement in vistir def normalize_path(path): return os.path.normpath(os.path.normcase( os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) @@ -128,9 +126,10 @@ def filter_pythons(path): path = vistir.compat.Path(str(path)) if not path.is_dir(): return path if path_is_python(path) else None - return filter(lambda x: path_is_python(x), path.iterdir()) + return filter(path_is_python, path.iterdir()) +# TODO: Port to vistir def unnest(item): if isinstance(item, Iterable) and not isinstance(item, six.string_types): item, target = itertools.tee(item, 2) @@ -168,5 +167,6 @@ def parse_asdf_version_order(filename=".tool-versions"): return versions.split() +# TODO: Reimplement in vistir def is_in_path(path, parent): return normalize_path(str(path)).startswith(normalize_path(str(parent))) diff --git a/pipenv/vendor/pytoml/LICENSE b/pipenv/vendor/pytoml/LICENSE deleted file mode 100644 index 9739fc67..00000000 --- a/pipenv/vendor/pytoml/LICENSE +++ /dev/null @@ -1,16 +0,0 @@ -No-notice MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/pipenv/vendor/pytoml/__init__.py b/pipenv/vendor/pytoml/__init__.py deleted file mode 100644 index 8dc73155..00000000 --- a/pipenv/vendor/pytoml/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .core import TomlError -from .parser import load, loads -from .writer import dump, dumps diff --git a/pipenv/vendor/pytoml/core.py b/pipenv/vendor/pytoml/core.py deleted file mode 100644 index c182734e..00000000 --- a/pipenv/vendor/pytoml/core.py +++ /dev/null @@ -1,13 +0,0 @@ -class TomlError(RuntimeError): - def __init__(self, message, line, col, filename): - RuntimeError.__init__(self, message, line, col, filename) - self.message = message - self.line = line - self.col = col - self.filename = filename - - def __str__(self): - return '{}({}, {}): {}'.format(self.filename, self.line, self.col, self.message) - - def __repr__(self): - return 'TomlError({!r}, {!r}, {!r}, {!r})'.format(self.message, self.line, self.col, self.filename) diff --git a/pipenv/vendor/pytoml/parser.py b/pipenv/vendor/pytoml/parser.py deleted file mode 100644 index 9f94e923..00000000 --- a/pipenv/vendor/pytoml/parser.py +++ /dev/null @@ -1,374 +0,0 @@ -import string, re, sys, datetime -from .core import TomlError - -if sys.version_info[0] == 2: - _chr = unichr -else: - _chr = chr - -def load(fin, translate=lambda t, x, v: v, object_pairs_hook=dict): - return loads(fin.read(), translate=translate, object_pairs_hook=object_pairs_hook, filename=getattr(fin, 'name', repr(fin))) - -def loads(s, filename='', translate=lambda t, x, v: v, object_pairs_hook=dict): - if isinstance(s, bytes): - s = s.decode('utf-8') - - s = s.replace('\r\n', '\n') - - root = object_pairs_hook() - tables = object_pairs_hook() - scope = root - - src = _Source(s, filename=filename) - ast = _p_toml(src, object_pairs_hook=object_pairs_hook) - - def error(msg): - raise TomlError(msg, pos[0], pos[1], filename) - - def process_value(v, object_pairs_hook): - kind, text, value, pos = v - if kind == 'str' and value.startswith('\n'): - value = value[1:] - if kind == 'array': - if value and any(k != value[0][0] for k, t, v, p in value[1:]): - error('array-type-mismatch') - value = [process_value(item, object_pairs_hook=object_pairs_hook) for item in value] - elif kind == 'table': - value = object_pairs_hook([(k, process_value(value[k], object_pairs_hook=object_pairs_hook)) for k in value]) - return translate(kind, text, value) - - for kind, value, pos in ast: - if kind == 'kv': - k, v = value - if k in scope: - error('duplicate_keys. Key "{0}" was used more than once.'.format(k)) - scope[k] = process_value(v, object_pairs_hook=object_pairs_hook) - else: - is_table_array = (kind == 'table_array') - cur = tables - for name in value[:-1]: - if isinstance(cur.get(name), list): - d, cur = cur[name][-1] - else: - d, cur = cur.setdefault(name, (None, object_pairs_hook())) - - scope = object_pairs_hook() - name = value[-1] - if name not in cur: - if is_table_array: - cur[name] = [(scope, object_pairs_hook())] - else: - cur[name] = (scope, object_pairs_hook()) - elif isinstance(cur[name], list): - if not is_table_array: - error('table_type_mismatch') - cur[name].append((scope, object_pairs_hook())) - else: - if is_table_array: - error('table_type_mismatch') - old_scope, next_table = cur[name] - if old_scope is not None: - error('duplicate_tables') - cur[name] = (scope, next_table) - - def merge_tables(scope, tables): - if scope is None: - scope = object_pairs_hook() - for k in tables: - if k in scope: - error('key_table_conflict') - v = tables[k] - if isinstance(v, list): - scope[k] = [merge_tables(sc, tbl) for sc, tbl in v] - else: - scope[k] = merge_tables(v[0], v[1]) - return scope - - return merge_tables(root, tables) - -class _Source: - def __init__(self, s, filename=None): - self.s = s - self._pos = (1, 1) - self._last = None - self._filename = filename - self.backtrack_stack = [] - - def last(self): - return self._last - - def pos(self): - return self._pos - - def fail(self): - return self._expect(None) - - def consume_dot(self): - if self.s: - self._last = self.s[0] - self.s = self[1:] - self._advance(self._last) - return self._last - return None - - def expect_dot(self): - return self._expect(self.consume_dot()) - - def consume_eof(self): - if not self.s: - self._last = '' - return True - return False - - def expect_eof(self): - return self._expect(self.consume_eof()) - - def consume(self, s): - if self.s.startswith(s): - self.s = self.s[len(s):] - self._last = s - self._advance(s) - return True - return False - - def expect(self, s): - return self._expect(self.consume(s)) - - def consume_re(self, re): - m = re.match(self.s) - if m: - self.s = self.s[len(m.group(0)):] - self._last = m - self._advance(m.group(0)) - return m - return None - - def expect_re(self, re): - return self._expect(self.consume_re(re)) - - def __enter__(self): - self.backtrack_stack.append((self.s, self._pos)) - - def __exit__(self, type, value, traceback): - if type is None: - self.backtrack_stack.pop() - else: - self.s, self._pos = self.backtrack_stack.pop() - return type == TomlError - - def commit(self): - self.backtrack_stack[-1] = (self.s, self._pos) - - def _expect(self, r): - if not r: - raise TomlError('msg', self._pos[0], self._pos[1], self._filename) - return r - - def _advance(self, s): - suffix_pos = s.rfind('\n') - if suffix_pos == -1: - self._pos = (self._pos[0], self._pos[1] + len(s)) - else: - self._pos = (self._pos[0] + s.count('\n'), len(s) - suffix_pos) - -_ews_re = re.compile(r'(?:[ \t]|#[^\n]*\n|#[^\n]*\Z|\n)*') -def _p_ews(s): - s.expect_re(_ews_re) - -_ws_re = re.compile(r'[ \t]*') -def _p_ws(s): - s.expect_re(_ws_re) - -_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', '\'': '\'', - '\\': '\\', '/': '/', 'f': '\f' } - -_basicstr_re = re.compile(r'[^"\\\000-\037]*') -_short_uni_re = re.compile(r'u([0-9a-fA-F]{4})') -_long_uni_re = re.compile(r'U([0-9a-fA-F]{8})') -_escapes_re = re.compile('[bnrt"\'\\\\/f]') -_newline_esc_re = re.compile('\n[ \t\n]*') -def _p_basicstr_content(s, content=_basicstr_re): - res = [] - while True: - res.append(s.expect_re(content).group(0)) - if not s.consume('\\'): - break - if s.consume_re(_newline_esc_re): - pass - elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re): - res.append(_chr(int(s.last().group(1), 16))) - else: - s.expect_re(_escapes_re) - res.append(_escapes[s.last().group(0)]) - return ''.join(res) - -_key_re = re.compile(r'[0-9a-zA-Z-_]+') -def _p_key(s): - with s: - s.expect('"') - r = _p_basicstr_content(s, _basicstr_re) - s.expect('"') - return r - if s.consume('\''): - if s.consume('\'\''): - r = s.expect_re(_litstr_ml_re).group(0) - s.expect('\'\'\'') - else: - r = s.expect_re(_litstr_re).group(0) - s.expect('\'') - return r - return s.expect_re(_key_re).group(0) - -_float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?') -_datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') - -_basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') -_litstr_re = re.compile(r"[^'\000\010\012-\037]*") -_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") -def _p_value(s, object_pairs_hook): - pos = s.pos() - - if s.consume('true'): - return 'bool', s.last(), True, pos - if s.consume('false'): - return 'bool', s.last(), False, pos - - if s.consume('"'): - if s.consume('""'): - r = _p_basicstr_content(s, _basicstr_ml_re) - s.expect('"""') - else: - r = _p_basicstr_content(s, _basicstr_re) - s.expect('"') - return 'str', r, r, pos - - if s.consume('\''): - if s.consume('\'\''): - r = s.expect_re(_litstr_ml_re).group(0) - s.expect('\'\'\'') - else: - r = s.expect_re(_litstr_re).group(0) - s.expect('\'') - return 'str', r, r, pos - - if s.consume_re(_datetime_re): - m = s.last() - s0 = m.group(0) - r = map(int, m.groups()[:6]) - if m.group(7): - micro = float(m.group(7)) - else: - micro = 0 - - if m.group(8): - g = int(m.group(8), 10) * 60 + int(m.group(9), 10) - tz = _TimeZone(datetime.timedelta(0, g * 60)) - else: - tz = _TimeZone(datetime.timedelta(0, 0)) - - y, m, d, H, M, S = r - dt = datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) - return 'datetime', s0, dt, pos - - if s.consume_re(_float_re): - m = s.last().group(0) - r = m.replace('_','') - if '.' in m or 'e' in m or 'E' in m: - return 'float', m, float(r), pos - else: - return 'int', m, int(r, 10), pos - - if s.consume('['): - items = [] - with s: - while True: - _p_ews(s) - items.append(_p_value(s, object_pairs_hook=object_pairs_hook)) - s.commit() - _p_ews(s) - s.expect(',') - s.commit() - _p_ews(s) - s.expect(']') - return 'array', None, items, pos - - if s.consume('{'): - _p_ws(s) - items = object_pairs_hook() - if not s.consume('}'): - k = _p_key(s) - _p_ws(s) - s.expect('=') - _p_ws(s) - items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) - _p_ws(s) - while s.consume(','): - _p_ws(s) - k = _p_key(s) - _p_ws(s) - s.expect('=') - _p_ws(s) - items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) - _p_ws(s) - s.expect('}') - return 'table', None, items, pos - - s.fail() - -def _p_stmt(s, object_pairs_hook): - pos = s.pos() - if s.consume( '['): - is_array = s.consume('[') - _p_ws(s) - keys = [_p_key(s)] - _p_ws(s) - while s.consume('.'): - _p_ws(s) - keys.append(_p_key(s)) - _p_ws(s) - s.expect(']') - if is_array: - s.expect(']') - return 'table_array' if is_array else 'table', keys, pos - - key = _p_key(s) - _p_ws(s) - s.expect('=') - _p_ws(s) - value = _p_value(s, object_pairs_hook=object_pairs_hook) - return 'kv', (key, value), pos - -_stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*') -def _p_toml(s, object_pairs_hook): - stmts = [] - _p_ews(s) - with s: - stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) - while True: - s.commit() - s.expect_re(_stmtsep_re) - stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) - _p_ews(s) - s.expect_eof() - return stmts - -class _TimeZone(datetime.tzinfo): - def __init__(self, offset): - self._offset = offset - - def utcoffset(self, dt): - return self._offset - - def dst(self, dt): - return None - - def tzname(self, dt): - m = self._offset.total_seconds() // 60 - if m < 0: - res = '-' - m = -m - else: - res = '+' - h = m // 60 - m = m - h * 60 - return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/pipenv/vendor/pytoml/writer.py b/pipenv/vendor/pytoml/writer.py deleted file mode 100644 index 6eaf5d76..00000000 --- a/pipenv/vendor/pytoml/writer.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import unicode_literals -import io, datetime, math, sys - -if sys.version_info[0] == 3: - long = int - unicode = str - - -def dumps(obj, sort_keys=False): - fout = io.StringIO() - dump(obj, fout, sort_keys=sort_keys) - return fout.getvalue() - - -_escapes = {'\n': 'n', '\r': 'r', '\\': '\\', '\t': 't', '\b': 'b', '\f': 'f', '"': '"'} - - -def _escape_string(s): - res = [] - start = 0 - - def flush(): - if start != i: - res.append(s[start:i]) - return i + 1 - - i = 0 - while i < len(s): - c = s[i] - if c in '"\\\n\r\t\b\f': - start = flush() - res.append('\\' + _escapes[c]) - elif ord(c) < 0x20: - start = flush() - res.append('\\u%04x' % ord(c)) - i += 1 - - flush() - return '"' + ''.join(res) + '"' - - -def _escape_id(s): - if any(not c.isalnum() and c not in '-_' for c in s): - return _escape_string(s) - return s - - -def _format_list(v): - return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) - -# Formula from: -# https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds -# Once support for py26 is dropped, this can be replaced by td.total_seconds() -def _total_seconds(td): - return ((td.microseconds - + (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6) - -def _format_value(v): - if isinstance(v, bool): - return 'true' if v else 'false' - if isinstance(v, int) or isinstance(v, long): - return unicode(v) - if isinstance(v, float): - if math.isnan(v) or math.isinf(v): - raise ValueError("{0} is not a valid TOML value".format(v)) - else: - return repr(v) - elif isinstance(v, unicode) or isinstance(v, bytes): - return _escape_string(v) - elif isinstance(v, datetime.datetime): - offs = v.utcoffset() - offs = _total_seconds(offs) // 60 if offs is not None else 0 - - if offs == 0: - suffix = 'Z' - else: - if offs > 0: - suffix = '+' - else: - suffix = '-' - offs = -offs - suffix = '{0}{1:.02}{2:.02}'.format(suffix, offs // 60, offs % 60) - - if v.microsecond: - return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix - else: - return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix - elif isinstance(v, list): - return _format_list(v) - else: - raise RuntimeError(v) - - -def dump(obj, fout, sort_keys=False): - tables = [((), obj, False)] - - while tables: - name, table, is_array = tables.pop() - if name: - section_name = '.'.join(_escape_id(c) for c in name) - if is_array: - fout.write('[[{0}]]\n'.format(section_name)) - else: - fout.write('[{0}]\n'.format(section_name)) - - table_keys = sorted(table.keys()) if sort_keys else table.keys() - new_tables = [] - has_kv = False - for k in table_keys: - v = table[k] - if isinstance(v, dict): - new_tables.append((name + (k,), v, False)) - elif isinstance(v, list) and v and all(isinstance(o, dict) for o in v): - new_tables.extend((name + (k,), d, True) for d in v) - elif v is None: - # based on mojombo's comment: https://github.com/toml-lang/toml/issues/146#issuecomment-25019344 - fout.write( - '#{} = null # To use: uncomment and replace null with value\n'.format(_escape_id(k))) - has_kv = True - else: - fout.write('{0} = {1}\n'.format(_escape_id(k), _format_value(v))) - has_kv = True - - tables.extend(reversed(new_tables)) - - if (name or has_kv) and tables: - fout.write('\n') diff --git a/pipenv/vendor/requests/__version__.py b/pipenv/vendor/requests/__version__.py index be8a45fe..803773a0 100644 --- a/pipenv/vendor/requests/__version__.py +++ b/pipenv/vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.20.0' -__build__ = 0x022000 +__version__ = '2.20.1' +__build__ = 0x022001 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/pipenv/vendor/requests/sessions.py b/pipenv/vendor/requests/sessions.py index a448bd83..d73d700f 100644 --- a/pipenv/vendor/requests/sessions.py +++ b/pipenv/vendor/requests/sessions.py @@ -19,7 +19,7 @@ from .cookies import ( from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook from ._internal_utils import to_native_string -from .utils import to_key_val_list, default_headers +from .utils import to_key_val_list, default_headers, DEFAULT_PORTS from .exceptions import ( TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) @@ -128,8 +128,17 @@ class SessionRedirectMixin(object): if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): return False + + # Handle default port usage corresponding to scheme. + changed_port = old_parsed.port != new_parsed.port + changed_scheme = old_parsed.scheme != new_parsed.scheme + default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None) + if (not changed_scheme and old_parsed.port in default_port + and new_parsed.port in default_port): + return False + # Standard case: root URI must match - return old_parsed.port != new_parsed.port or old_parsed.scheme != new_parsed.scheme + return changed_port or changed_scheme def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): diff --git a/pipenv/vendor/requests/utils.py b/pipenv/vendor/requests/utils.py index 0ce7fe11..8170a8d2 100644 --- a/pipenv/vendor/requests/utils.py +++ b/pipenv/vendor/requests/utils.py @@ -38,6 +38,8 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() +DEFAULT_PORTS = {'http': 80, 'https': 443} + if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups @@ -264,7 +266,7 @@ def from_key_val_list(value): >>> from_key_val_list([('key', 'val')]) OrderedDict([('key', 'val')]) >>> from_key_val_list('string') - ValueError: need more than 1 value to unpack + ValueError: cannot encode objects that are not 2-tuples >>> from_key_val_list({'key': 'val'}) OrderedDict([('key', 'val')]) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index f6c985d3..32415c61 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.2.7' +__version__ = '1.3.1' import logging import warnings diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/vendor/requirementslib/exceptions.py index 23bc5e50..17b884eb 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/vendor/requirementslib/exceptions.py @@ -24,7 +24,7 @@ class RequirementError(Exception): class MissingParameter(Exception): def __init__(self, param): - super(Exception, self).__init__() + Exception.__init__(self) print("Missing parameter: %s" % param, file=sys.stderr, flush=True) @@ -43,7 +43,7 @@ class FileCorruptException(OSError): self.path = path self.backup_path = backup_path self.show(self.path, self.backup_path) - super(OSError, self).__init__(path, *args, **kwargs) + OSError.__init__(self, path, *args, **kwargs) @classmethod def show(cls, path, backup_path=None): diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index dbb024be..0f6de6bf 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -53,17 +53,19 @@ class PipfileLoader(plette.pipfiles.Pipfile): _data["source"] = _data.get("source", []) + _data.get("sources", []) _data = reorder_source_keys(_data) if "source" not in _data: - # HACK: There is no good way to prepend a section to an existing - # TOML document, but there's no good way to copy non-structural - # content from one TOML document to another either. Modify the - # TOML content directly, and load the new in-memory document. - sep = "" if content.startswith("\n") else "\n" - content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content + if "sources" in _data: + _data["source"] = _data["sources"] + content = tomlkit.dumps(_data) + else: + # HACK: There is no good way to prepend a section to an existing + # TOML document, but there's no good way to copy non-structural + # content from one TOML document to another either. Modify the + # TOML content directly, and load the new in-memory document. + sep = "" if content.startswith("\n") else "\n" + content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content data = tomlkit.loads(content) - data = reorder_source_keys(data) instance = cls(data) - new_data = reorder_source_keys(instance._data) - instance._data = new_data + instance._data = dict(instance._data) return instance def __getattribute__(self, key): diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 6107a240..006ee609 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -223,7 +223,6 @@ class SetupInfo(object): if self.setup_py is not None and self.setup_py.exists(): target_cwd = self.setup_py.parent.as_posix() with cd(target_cwd), _suppress_distutils_logs(): - from setuptools.dist import distutils script_name = self.setup_py.as_posix() args = ["egg_info", "--egg-base", self.base_dir] g = {"__file__": script_name, "__name__": "__main__"} @@ -306,7 +305,7 @@ class SetupInfo(object): if not self.requires or not self.name: try: self.run_setup() - except Exception as e: + except Exception: self.get_egg_metadata() if not self.requires or not self.name: self.get_egg_metadata() diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index 312efbb3..f3653e32 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -88,7 +88,7 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pip format""" + """Cleans VCS uris from pipenv.patched.notpip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index f3554d56..809c973c 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -31,7 +31,7 @@ from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfi from .spin import VistirSpinner, create_spinner -__version__ = "0.2.4" +__version__ = '0.2.4' __all__ = [ diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 3f191120..0920a9c3 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -63,7 +63,7 @@ def cd(path): >>> print(os.path.abspath(os.curdir)) '/home/user/code/myrepo' >>> with cd("/home/user/code/otherdir/subdir"): - print("Changed directory: %s" % os.path.abspath(os.curdir)) + ... print("Changed directory: %s" % os.path.abspath(os.curdir)) Changed directory: /home/user/code/otherdir/subdir >>> print(os.path.abspath(os.curdir)) '/home/user/code/myrepo' diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 23ae0252..febaddbc 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, unicode_literals, print_function import atexit import errno @@ -275,9 +275,12 @@ def set_write_bit(fn): file_stat = os.stat(fn).st_mode os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) if not os.path.isdir(fn): - return + try: + os.chflags(fn, 0) + except AttributeError: + pass for root, dirs, files in os.walk(fn, topdown=False): - for dir_ in [os.path.join(root, d) for d in dirs]: + for dir_ in [os.path.join(root,d) for d in dirs]: set_write_bit(dir_) for file_ in [os.path.join(root, f) for f in files]: set_write_bit(file_) From 5e5e1ed4bc0e09c3a45394c2369502e0ead217c2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 12:23:58 -0500 Subject: [PATCH 190/218] Stray print Signed-off-by: Dan Ryan --- pipenv/cli/command.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 75a3b966..14120e35 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -226,7 +226,6 @@ def install( ): """Installs provided packages and adds them to Pipfile, or (if no packages are given), installs all packages from Pipfile.""" from ..core import do_install - echo("Skip lock value: %s" % state.installstate.skip_lock) retcode = do_install( dev=state.installstate.dev, From 08d94d4ebc7f21e49ac1781c73d2c03433ac730d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 16:53:20 -0500 Subject: [PATCH 191/218] Update core and rebuild ci Signed-off-by: Dan Ryan --- pipenv/core.py | 25 ++++++++++++++++--------- pipenv/exceptions.py | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 8504553c..d3e9002c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -36,8 +36,6 @@ from .utils import ( download_file, is_pinned, is_star, - rmtree, - clean_resolved_dep, parse_indexes, escape_cmd, create_spinner, @@ -135,8 +133,8 @@ def do_clear(): from pip import locations try: - shutil.rmtree(PIPENV_CACHE_DIR) - shutil.rmtree(locations.USER_CACHE_DIR) + vistir.path.rmtree(PIPENV_CACHE_DIR) + vistir.path.rmtree(locations.USER_CACHE_DIR) except OSError as e: # Ignore FileNotFoundError. This is needed for Python 2.7. import errno @@ -186,7 +184,7 @@ def cleanup_virtualenv(bare=True): click.echo(crayons.red("Environment creation aborted.")) try: # Delete the virtualenv. - rmtree(project.virtualenv_location) + vistir.path.rmtree(project.virtualenv_location) except OSError as e: click.echo( "{0} An error occurred while removing {1}!".format( @@ -657,8 +655,8 @@ def do_where(virtualenv=False, bare=True): def _cleanup_procs(procs, concurrent, failed_deps_queue, retry=True): while not procs.empty(): c = procs.get() - # if concurrent: - c.block() + if concurrent: + c.block() failed = False if c.return_code != 0: failed = True @@ -740,6 +738,8 @@ def batch_install(deps_list, procs, failed_deps_queue, trusted_hosts=trusted_hosts, extra_indexes=extra_indexes ) + if dep.is_vcs: + c.block() if procs.qsize() < nprocs: c.dep = dep procs.put(c) @@ -760,7 +760,8 @@ def do_install_dependencies( requirements_dir=None, pypi_mirror=False, ): - """"Executes the install functionality. + """" + Executes the install functionality. If requirements is True, simply spits out a requirements format to stdout. """ @@ -1060,6 +1061,12 @@ def do_lock( lockfile[section_name][canonical_name] = cached_lockfile[ section_name ][canonical_name].copy() + for key in ["default", "develop"]: + packages = set(cached_lockfile[key].keys()) + new_lockfile = set(lockfile[key].keys()) + missing = packages - new_lockfile + for missing_pkg in missing: + lockfile[key][missing_pkg] = cached_lockfile[key][missing_pkg].copy() # Overwrite any develop packages with default packages. lockfile["develop"].update(overwrite_dev(lockfile.get("default", {}), lockfile["develop"])) if write: @@ -1085,7 +1092,7 @@ def do_purge(bare=False, downloads=False, allow_global=False): if downloads: if not bare: click.echo(crayons.normal(fix_utf8("Clearing out downloads directory…"), bold=True)) - shutil.rmtree(project.download_location) + vistir.path.rmtree(project.download_location) return # Remove comments from the output, if any. diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 3fda8021..62e25d53 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -202,7 +202,7 @@ class PipfileException(PipenvFileError): class SetupException(PipenvException): def __init__(self, message=None, **kwargs): - PipenvException.__init__(message, **kwargs) + PipenvException.__init__(self, message, **kwargs) class VirtualenvException(PipenvException): From a8f4c7e2a3839667a8ff0e0626768e635b44004a Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 16:56:06 -0500 Subject: [PATCH 192/218] accidental commit Signed-off-by: Dan Ryan --- pipenv/core.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index d3e9002c..f815e5de 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1061,12 +1061,6 @@ def do_lock( lockfile[section_name][canonical_name] = cached_lockfile[ section_name ][canonical_name].copy() - for key in ["default", "develop"]: - packages = set(cached_lockfile[key].keys()) - new_lockfile = set(lockfile[key].keys()) - missing = packages - new_lockfile - for missing_pkg in missing: - lockfile[key][missing_pkg] = cached_lockfile[key][missing_pkg].copy() # Overwrite any develop packages with default packages. lockfile["develop"].update(overwrite_dev(lockfile.get("default", {}), lockfile["develop"])) if write: From 41cbe4dbdd927206cca6d07150b73f5eb434baff Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 17:01:30 -0500 Subject: [PATCH 193/218] Pin postreleases to pass packaging ci Signed-off-by: Dan Ryan --- pipenv/vendor/vendor.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 0a98277b..ff7226b2 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -21,13 +21,13 @@ pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.9 +pythonfinder==1.1.9.post1 requests==2.20.1 chardet==3.0.4 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.3.1 +requirementslib==1.3.1.post1 attrs==18.2.0 distlib==0.2.8 packaging==18.0 From a84f5443af8a3e6615acd1a4afebf96d1af8002d Mon Sep 17 00:00:00 2001 From: Peter Schutt Date: Thu, 15 Nov 2018 10:17:58 +1100 Subject: [PATCH 194/218] Incorrect ref in VCS example Small changes, but a couple of issues in the `A Note About VCS Dependencies` section of the docs. Running the example `$ pipenv install -e git+https://github.com/requests/requests.git@v2.19#egg=requests` raises the error: `Did not find branch or tag 'v2.19', assuming revision or ref.` I think it should be 'v2.19.1' which is just a typo as 'v2.19.1' is used in the subsequent lines of the example. The example Pipfile in the section excludes the `v` in the version number written to the `ref = ` part. I found it a little confusing that it would strip the 'v' from the tag, but then on testing it I found that it doesn't do that: ``` [packages] requests = {editable = true, ref = "v2.20.1", git = "https://github.com/requests/requests.git"} ``` Also, while I was here I figured I'd update the example to the most recent requests release. Thanks. --- docs/basics.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/basics.rst b/docs/basics.rst index ee2e6621..00f4c7d9 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -367,18 +367,18 @@ The only optional section is the ``@`` section. When using git o Note that it is **strongly recommended** that you install any version-controlled dependencies in editable mode, using ``pipenv install -e``, in order to ensure that dependency resolution can be performed with an up to date copy of the repository each time it is performed, and that it includes all known dependencies. -Below is an example usage which installs the git repository located at ``https://github.com/requests/requests.git`` from tag ``v2.19.1`` as package name ``requests``:: +Below is an example usage which installs the git repository located at ``https://github.com/requests/requests.git`` from tag ``v2.20.1`` as package name ``requests``:: - $ pipenv install -e git+https://github.com/requests/requests.git@v2.19#egg=requests + $ pipenv install -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests Creating a Pipfile for this project... - Installing -e git+https://github.com/requests/requests.git@v2.19.1#egg=requests... + Installing -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests... [...snipped...] - Adding -e git+https://github.com/requests/requests.git@v2.19.1#egg=requests to Pipfile's [packages]... + Adding -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests to Pipfile's [packages]... [...] $ cat Pipfile [packages] - requests = {git = "https://github.com/requests/requests.git", editable = true, ref = "2.19.1"} + requests = {git = "https://github.com/requests/requests.git", editable = true, ref = "v2.20.1"} Valid values for ```` include ``git``, ``bzr``, ``svn``, and ``hg``. Valid values for ```` include ``http``, ``https``, ``ssh``, and ``file``. In specific cases you also have access to other schemes: ``svn`` may be combined with ``svn`` as a scheme, and ``bzr`` can be combined with ``sftp`` and ``lp``. From b51afde84abe4f3bdfaff580c5a506092061527f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 18:25:09 -0500 Subject: [PATCH 195/218] Update towncrier template Signed-off-by: Dan Ryan --- news/towncrier_template.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/news/towncrier_template.rst b/news/towncrier_template.rst index 37d89a60..55505f82 100644 --- a/news/towncrier_template.rst +++ b/news/towncrier_template.rst @@ -13,9 +13,7 @@ {% if definitions[category]['showcontent'] %} {% for text, values in sections[section][category]|dictsort(by='value') %} -- {{ text }}{% if category != 'process' %} - {{ values|sort|join(',\n ') }} - {% endif %} +- {{ text }}{% if category != 'process' %}{{ values|sort|join(',\n ') }}{% endif %} {% endfor %} {% else %} From 6f07e72e5aeaa642f09b3517f8816b82675dd141 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 18:35:38 -0500 Subject: [PATCH 196/218] Release 2018.11.14 Signed-off-by: Dan Ryan --- CHANGELOG.rst | 86 +++++++++++++++++++ Pipfile.lock | 192 ++++++++++++++++++++++++++---------------- news/1977.feature.rst | 1 - news/2200.feature.rst | 1 - news/2394.bugfix.rst | 1 - news/2411.feature.rst | 1 - news/2737.bugfix.rst | 1 - news/2825.feature | 2 - news/2828.feature.rst | 1 - news/2848.bugfix.rst | 1 - news/2885.bugfix.rst | 1 - news/2983.bugfix.rst | 1 - news/2996.trivial | 1 - news/3020.feature.rst | 1 - news/3026.bugfix | 1 - news/3041.feature | 1 - news/3047.bugfix.rst | 1 - news/3055.bugfix.rst | 1 - news/3061.vendor.rst | 1 - news/3069.trivial | 1 - news/3074.doc.rst | 1 - news/3088.bugfix.rst | 1 - news/3089.feature.rst | 1 - news/3089.vendor.rst | 11 --- news/3090.bugfix.rst | 1 - news/3094.bugfix.rst | 1 - news/3096.bugfix.rst | 1 - news/3096.feature.rst | 1 - news/3096.vendor.rst | 6 -- news/3099.bugfix.rst | 1 - news/3102.bugfix.rst | 1 - news/3104.bugfix | 1 - news/3106.bugfix.rst | 1 - news/3109.bugfix.rst | 1 - news/3113.bugfix.rst | 1 - news/3114.bugfix.rst | 1 - news/3117.bugfix.rst | 1 - news/3121.bugfix.rst | 1 - news/3121.vendor.rst | 1 - news/3131.bugfix.rst | 1 - news/3134.bugfix.rst | 1 - news/3145.bugfix.rst | 1 - news/3151.bugfix | 1 - news/3154.trivial.rst | 1 - news/3158.bugfix.rst | 1 - news/3158.feature.rst | 1 - news/3164.bugfix.rst | 1 - news/3170.feature | 1 - news/3179.trivial | 1 - news/3183.bugfix.rst | 1 - news/3185.bugfix.rst | 1 - news/3191.vendor.rst | 1 - news/3196.vendor.rst | 1 - news/3199.trivial | 1 - news/3201.bugfix | 1 - news/3216.bugfix.rst | 1 - news/3217.feature.rst | 1 - pipenv/__version__.py | 2 +- 58 files changed, 208 insertions(+), 143 deletions(-) delete mode 100644 news/1977.feature.rst delete mode 100644 news/2200.feature.rst delete mode 100644 news/2394.bugfix.rst delete mode 100644 news/2411.feature.rst delete mode 100644 news/2737.bugfix.rst delete mode 100644 news/2825.feature delete mode 100644 news/2828.feature.rst delete mode 100644 news/2848.bugfix.rst delete mode 100644 news/2885.bugfix.rst delete mode 100644 news/2983.bugfix.rst delete mode 100644 news/2996.trivial delete mode 100644 news/3020.feature.rst delete mode 100644 news/3026.bugfix delete mode 100644 news/3041.feature delete mode 100644 news/3047.bugfix.rst delete mode 100644 news/3055.bugfix.rst delete mode 100644 news/3061.vendor.rst delete mode 100644 news/3069.trivial delete mode 100644 news/3074.doc.rst delete mode 100644 news/3088.bugfix.rst delete mode 100644 news/3089.feature.rst delete mode 100644 news/3089.vendor.rst delete mode 100644 news/3090.bugfix.rst delete mode 100644 news/3094.bugfix.rst delete mode 100644 news/3096.bugfix.rst delete mode 100644 news/3096.feature.rst delete mode 100644 news/3096.vendor.rst delete mode 100644 news/3099.bugfix.rst delete mode 100644 news/3102.bugfix.rst delete mode 100644 news/3104.bugfix delete mode 100644 news/3106.bugfix.rst delete mode 100644 news/3109.bugfix.rst delete mode 100644 news/3113.bugfix.rst delete mode 100644 news/3114.bugfix.rst delete mode 100644 news/3117.bugfix.rst delete mode 100644 news/3121.bugfix.rst delete mode 100644 news/3121.vendor.rst delete mode 100644 news/3131.bugfix.rst delete mode 100644 news/3134.bugfix.rst delete mode 100644 news/3145.bugfix.rst delete mode 100644 news/3151.bugfix delete mode 100644 news/3154.trivial.rst delete mode 100644 news/3158.bugfix.rst delete mode 100644 news/3158.feature.rst delete mode 100644 news/3164.bugfix.rst delete mode 100644 news/3170.feature delete mode 100644 news/3179.trivial delete mode 100644 news/3183.bugfix.rst delete mode 100644 news/3185.bugfix.rst delete mode 100644 news/3191.vendor.rst delete mode 100644 news/3196.vendor.rst delete mode 100644 news/3199.trivial delete mode 100644 news/3201.bugfix delete mode 100644 news/3216.bugfix.rst delete mode 100644 news/3217.feature.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ec8eaef1..3686ca85 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,89 @@ +2018.11.14 (2018-11-14) +======================= + +Features & Improvements +----------------------- + +- Improved exceptions and error handling on failures.`#1977 `_ +- Added persistent settings for all CLI flags via ``PIPENV_{FLAG_NAME}`` environment variables by enabling ``auto_envvar_prefix=PIPENV`` in click (implements PEEP-0002).`#2200 `_ +- Added improved messaging about available but skipped updates due to dependency conflicts when running ``pipenv update --outdated``.`#2411 `_ +- Added environment variable `PIPENV_PYUP_API_KEY` to add ability + to override the bundled pyup.io API key.`#2825 `_ +- Added additional output to ``pipenv update --outdated`` to indicate that the operation succeded and all packages were already up to date.`#2828 `_ +- Updated ``crayons`` patch to enable colors on native powershell but swap native blue for magenta.`#3020 `_ +- Added support for ``--bare`` to ``pipenv clean``, and fixed ``pipenv sync --bare`` to actually reduce output.`#3041 `_ +- Added windows-compatible spinner via upgraded ``vistir`` dependency.`#3089 `_ +- - Added support for python installations managed by ``asdf``.`#3096 `_ +- Improved runtime performance of no-op commands such as ``pipenv --venv`` by around 2/3.`#3158 `_ +- Do not show error but success for running ``pipenv uninstall --all`` in a fresh virtual environment.`#3170 `_ +- Improved asynchronous installation and error handling via queued subprocess paralleization.`#3217 `_ + +Bug Fixes +--------- + +- Remote non-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``.`#2394 `_ +- Non-ascii characters will now be handled correctly when parsed by pipenv's ``ToML`` parsers.`#2737 `_ +- Updated ``pipenv uninstall`` to respect the ``--skip-lock`` argument.`#2848 `_ +- Fixed a bug which caused uninstallation to sometimes fail to successfullly remove packages from ``Pipfiles`` with comments on preceding or following lines.`#2885 `_, + `#3099 `_ +- Pipenv will no longer fail when encountering python versions on Windows that have been uninstalled.`#2983 `_ +- Fixed unnecessary extras are added when translating markers`#3026 `_ +- Fixed a virtualenv creation issue which could cause new virtualenvs to inadvertently attempt to read and write to global site packages.`#3047 `_ +- Fixed an issue with virtualenv path derivation which could cause errors, particularly for users on WSL bash.`#3055 `_ +- Fixed a bug which caused ``Unexpected EOF`` errors to be thrown when ``pip`` was waiting for input from users who had put login credentials in environment variables.`#3088 `_ +- Fixed a bug in ``requirementslib`` which prevented successful installation from mercurial repositories.`#3090 `_ +- Fixed random resource warnings when using pyenv or any other subprocess calls.`#3094 `_ +- - Fixed a bug which sometimes prevented cloning and parsing ``mercurial`` requirements.`#3096 `_ +- Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors.`#3102 `_, + `#3114 `_, + `#3117 `_ +- Fix the path casing issue that makes `pipenv clean` fail on Windows`#3104 `_ +- Pipenv will avoid leaving build artifacts in the current working directory.`#3106 `_ +- Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures.`#3109 `_ +- Fixed an issue which caused ``pipenv clean`` to sometimes clean packages from the base ``site-packages`` folder or fail entirely.`#3113 `_ +- Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions.`#3121 `_ +- Added additional logic for ignoring and replacing non-ascii characters when formatting console output on non-UTF-8 systems.`#3131 `_ +- Fix virtual environment discovery when `PIPENV_VENV_IN_PROJECT` is set, but the in-project `.venv` is a file.`#3134 `_ +- Hashes for remote and local non-PyPI artifacts will now be included in ``Pipfile.lock`` during resolution.`#3145 `_ +- Fix project path hashing logic in purpose to prevent collisions of virtual environments.`#3151 `_ +- Fix package installation when the virtual environment path contains parentheses.`#3158 `_ +- Azure Pipelines YAML files are updated to use the latest syntax and product name.`#3164 `_ +- Fixed new spinner success message to write only one success message during resolution.`#3183 `_ +- Pipenv will now correctly respect the ``--pre`` option when used with ``pipenv install``.`#3185 `_ +- Fix a bug where exception is raised when run pipenv graph in a project without created virtualenv`#3201 `_ +- When sources are missing names, names will now be derived from the supplied URL.`#3216 `_ + +Vendored Libraries +------------------ + +- Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions.`#3061 `_, + `#3121 `_ +- Updated vendored dependencies: + - ``certifi 2018.08.24 => 2018.10.15`` + - ``urllib3 1.23 => 1.24`` + - ``requests 2.19.1 => 2.20.0`` + - ``shellingham ``1.2.6 => 1.2.7`` + - ``tomlkit 0.4.4. => 0.4.6`` + - ``vistir 0.1.6 => 0.1.8`` + - ``pythonfinder 0.1.2 => 0.1.3`` + - ``requirementslib 1.1.9 => 1.1.10`` + - ``backports.functools_lru_cache 1.5.0 (new)`` + - ``cursor 1.2.0 (new)```#3089 `_ +- Updated vendored dependencies: + - ``requests 2.19.1 => 2.20.1`` + - ``tomlkit 0.4.46 => 0.5.2`` + - ``vistir 0.1.6 => 0.2.4`` + - ``pythonfinder 1.1.2 => 1.1.8`` + - ``requirementslib 1.1.10 => 1.3.0```#3096 `_ +- Switch to ``tomlkit`` for parsing and writing. Drop ``prettytoml`` and ``contoml`` from vendors.`#3191 `_ +- Updated ``requirementslib`` to aid in resolution of local and remote archives.`#3196 `_ + +Improved Documentation +---------------------- + +- Expanded development and testing documentation for contributors to get started.`#3074 `_ + + 2018.10.13 (2018-10-13) ======================= diff --git a/Pipfile.lock b/Pipfile.lock index 30a618ac..f5e02ec8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -83,10 +83,10 @@ }, "bleach": { "hashes": [ - "sha256:9c471c0dd9c820f6bf4ee5ca3e348ceccefbc1475d9a40c397ed5d04e0b42c54", - "sha256:b407b2612b37e6cdc6704f84cec18c1f140b78e6c625652a844e89d6b9855f6b" + "sha256:48d39675b80a75f6d1c3bdbffec791cf0bbbab665cf01e20da701c77de278718", + "sha256:73d26f018af5d5adcdabf5c1c974add4361a9c76af215fe32fdec8a6fc5fb9b9" ], - "version": "==3.0.0" + "version": "==3.0.2" }, "cerberus": { "hashes": [ @@ -96,10 +96,10 @@ }, "certifi": { "hashes": [ - "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", - "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" + "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", + "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" ], - "version": "==2018.8.24" + "version": "==2018.10.15" }, "cffi": { "hashes": [ @@ -188,11 +188,10 @@ }, "colorama": { "hashes": [ - "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", - "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" + "sha256:a3d89af5db9e9806a779a50296b5fdb466e281147c2c235e8225ecc6dbf7bbf3", + "sha256:c9b54bebe91a6a803e0772c8561d53f2926bfeb17cd141fbabcb08424086595c" ], - "markers": "sys_platform == 'win32'", - "version": "==0.3.9" + "version": "==0.4.0" }, "configparser": { "hashes": [ @@ -201,6 +200,12 @@ "markers": "python_version < '3.2'", "version": "==3.5.0" }, + "cursor": { + "hashes": [ + "sha256:8ee9fe5b925e1001f6ae6c017e93682583d2b4d1ef7130a26cfcdf1651c0032c" + ], + "version": "==1.2.0" + }, "distlib": { "hashes": [ "sha256:57977cd7d9ea27986ec62f425630e4ddb42efe651ff80bc58ed8dbc3c7c21f19" @@ -270,6 +275,12 @@ "markers": "python_version < '3.3'", "version": "==1.0.2" }, + "future": { + "hashes": [ + "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + ], + "version": "==0.16.0" + }, "futures": { "hashes": [ "sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265", @@ -278,12 +289,6 @@ "markers": "python_version < '3' and python_version >= '2.6'", "version": "==3.2.0" }, - "future": { - "hashes": [ - "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" - ], - "version": "==0.16.0" - }, "idna": { "hashes": [ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", @@ -325,9 +330,10 @@ }, "itsdangerous": { "hashes": [ - "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" + "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", + "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" ], - "version": "==0.24" + "version": "==1.1.0" }, "jedi": { "hashes": [ @@ -346,9 +352,36 @@ }, "markupsafe": { "hashes": [ - "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665" + "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432", + "sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b", + "sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9", + "sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af", + "sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834", + "sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd", + "sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d", + "sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7", + "sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b", + "sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3", + "sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c", + "sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2", + "sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7", + "sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36", + "sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1", + "sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e", + "sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1", + "sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c", + "sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856", + "sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550", + "sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492", + "sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672", + "sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401", + "sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6", + "sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6", + "sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c", + "sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd", + "sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1" ], - "version": "==1.0" + "version": "==1.1.0" }, "mccabe": { "hashes": [ @@ -402,17 +435,24 @@ }, "pbr": { "hashes": [ - "sha256:1be135151a0da949af8c5d0ee9013d9eafada71237eb80b3ba8896b4f12ec5dc", - "sha256:cf36765bf2218654ae824ec8e14257259ba44e43b117fd573c8d07a9895adbdd" + "sha256:f59d71442f9ece3dffc17bc36575768e1ee9967756e6b6535f0ee1f0054c3d68", + "sha256:f6d5b23f226a2ba58e14e49aa3b1bfaf814d0199144b95d78458212444de1387" ], - "version": "==4.3.0" + "version": "==5.1.1" + }, + "pep517": { + "hashes": [ + "sha256:cc663a438fdfe2e88d8d3c5ef2203ac858de34e31b6609b1fc505d611490a926", + "sha256:f79bb08fb064dfc5b141204bfeb56a4141a6d504677fab4723036a464fc25cc1" + ], + "version": "==0.3" }, "pip-shims": { "hashes": [ - "sha256:164b93bc94b207613d9632f28f4d55eba9301f9454aaaba335de36c24d92d106", - "sha256:27e2439aa93af8c1b8e58cf63a40cbcd26959b26424904f2e6d57837af8f76c5" + "sha256:3bc24ec050a6b9eea35419467237e4f47eaf806dadc9999bf887355c377edea7", + "sha256:edb4cf3c509eab2f36b55c1ac1a59a4c485ccd537cc87934d74950880f641256" ], - "version": "==0.3.0" + "version": "==0.3.2" }, "pipenv": { "editable": true, @@ -426,6 +466,9 @@ "version": "==1.4.2" }, "plette": { + "extras": [ + "validation" + ], "hashes": [ "sha256:c0e3553c1e581d8423daccbd825789c6e7f29b7d9e00e5331b12e1642a1a26d3", "sha256:dde5d525cf5f0cbad4d938c83b93db17887918daf63c13eafed257c4f61b07b4" @@ -434,24 +477,24 @@ }, "pluggy": { "hashes": [ - "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", - "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" + "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", + "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f" ], - "version": "==0.7.1" + "version": "==0.8.0" }, "py": { "hashes": [ - "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1", - "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6" + "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", + "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" ], - "version": "==1.6.0" + "version": "==1.7.0" }, "pycodestyle": { "hashes": [ - "sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766", - "sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9" + "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83", + "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a" ], - "version": "==2.3.1" + "version": "==2.4.0" }, "pycparser": { "hashes": [ @@ -461,10 +504,10 @@ }, "pyflakes": { "hashes": [ - "sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f", - "sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805" + "sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49", + "sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae" ], - "version": "==1.6.0" + "version": "==2.0.0" }, "pygments": { "hashes": [ @@ -475,10 +518,10 @@ }, "pyparsing": { "hashes": [ - "sha256:bc6c7146b91af3f567cf6daeaec360bc07d45ffec4cf5353f4d7a208ce7ca30a", - "sha256:d29593d8ebe7b57d6967b62494f8c72b03ac0262b1eed63826c6f788b3606401" + "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b", + "sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592" ], - "version": "==2.2.2" + "version": "==2.3.0" }, "pytest": { "hashes": [ @@ -515,6 +558,12 @@ "index": "pypi", "version": "==1.23.2" }, + "pytoml": { + "hashes": [ + "sha256:ca2d0cb127c938b8b76a9a0d0f855cf930c1d50cc3a0af6d3595b566519a1013" + ], + "version": "==0.1.20" + }, "pytz": { "hashes": [ "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", @@ -525,17 +574,17 @@ }, "readme-renderer": { "hashes": [ - "sha256:237ca8705ffea849870de41101dba41543561da05c0ae45b2f1c547efa9843d2", - "sha256:f75049a3a7afa57165551e030dd8f9882ebf688b9600535a3f7e23596651875d" + "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f", + "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d" ], - "version": "==22.0" + "version": "==24.0" }, "requests": { "hashes": [ - "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", - "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" + "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", + "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" ], - "version": "==2.19.1" + "version": "==2.20.1" }, "requests-toolbelt": { "hashes": [ @@ -546,10 +595,10 @@ }, "requirementslib": { "hashes": [ - "sha256:39fb4aab3ebd7f46b266ddc98a3ac731127ee35fe6cf1b3e11be7c6551cc2c9b", - "sha256:810d8961f333d8fef92400f58b25f80003151fb424a545e244073fc3d95ae2dd" + "sha256:441a5bfa487d3f3f5fd5d81c27071d9fd36bb385f538b3a87d20556a80b76f76", + "sha256:89e1e02ff0b52ce9c610124eb990ae706e0aee08beef8c718e7b87e470cdceeb" ], - "version": "==1.1.7" + "version": "==1.3.1.post1" }, "resolvelib": { "hashes": [ @@ -638,10 +687,10 @@ }, "tqdm": { "hashes": [ - "sha256:18f1818ce951aeb9ea162ae1098b43f583f7d057b34d706f66939353d1208889", - "sha256:df02c0650160986bac0218bb07952245fc6960d23654648b5d5526ad5a4128c9" + "sha256:3c4d4a5a41ef162dd61f1edb86b0e1c7859054ab656b2e7c7b77e7fbf6d9f392", + "sha256:5b4d5549984503050883bc126280b386f5f4ca87e6c023c5d015655ad75bdebb" ], - "version": "==4.26.0" + "version": "==4.28.1" }, "twine": { "hashes": [ @@ -662,36 +711,37 @@ }, "urllib3": { "hashes": [ - "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", - "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" ], - "version": "==1.23" + "version": "==1.24.1" }, "virtualenv": { "hashes": [ - "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", - "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" + "sha256:686176c23a538ecc56d27ed9d5217abd34644823d6391cbeb232f42bf722baad", + "sha256:f899fafcd92e1150f40c8215328be38ff24b519cd95357fa6e78e006c7638208" ], - "version": "==16.0.0" + "version": "==16.1.0" }, "virtualenv-clone": { "hashes": [ - "sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7", - "sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8" + "sha256:afce268508aa5596c90dda234abe345deebc401a57d287bcbd76baa140a1aa58" ], - "version": "==0.3.0" + "version": "==0.4.0" }, "vistir": { - "hashes": [ - "sha256:8a360ac20cbcc0863d6dbbe7a52e8b2c9ebf48abd6833c3813a82c70708244af", - "sha256:bc6e10284792485c10585536e6aede9e38996c841cc9d2a67238cd05742c2d0b" + "extras": [ + "spinner" ], - "version": "==0.1.6" + "hashes": [ + "sha256:851bd783f2b85a372e563db741dc689cb9263ce2e067e387facdca0c36b6a6ea", + "sha256:b38ffc8ef83f85d81b4efa4cd31ea3bcd37bdb2bc9e8da9f20a40859bc44b57e" + ], + "version": "==0.2.4" }, "webencodings": { "hashes": [ - "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", - "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78" ], "version": "==0.5.1" }, @@ -704,10 +754,10 @@ }, "wheel": { "hashes": [ - "sha256:9fa1f772f1a2df2bd00ddb4fa57e1cc349301e1facb98fbe62329803a9ff1196", - "sha256:d215f4520a1ba1851a3c00ba2b4122665cd3d6b0834d2ba2816198b1e3024a0e" + "sha256:196c9842d79262bb66fcf59faa4bd0deb27da911dbc7c6cdca931080eb1f0783", + "sha256:c93e2d711f5f9841e17f53b0e6c0ff85593f3b416b6eec7a9452041a59a42688" ], - "version": "==0.32.1" + "version": "==0.32.2" }, "yaspin": { "hashes": [ diff --git a/news/1977.feature.rst b/news/1977.feature.rst deleted file mode 100644 index 33274c0f..00000000 --- a/news/1977.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Improved exceptions and error handling on failures. diff --git a/news/2200.feature.rst b/news/2200.feature.rst deleted file mode 100644 index daa4b215..00000000 --- a/news/2200.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added persistent settings for all CLI flags via ``PIPENV_{FLAG_NAME}`` environment variables by enabling ``auto_envvar_prefix=PIPENV`` in click (implements PEEP-0002). diff --git a/news/2394.bugfix.rst b/news/2394.bugfix.rst deleted file mode 100644 index 61ec9445..00000000 --- a/news/2394.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Remote non-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. diff --git a/news/2411.feature.rst b/news/2411.feature.rst deleted file mode 100644 index d99bfca8..00000000 --- a/news/2411.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added improved messaging about available but skipped updates due to dependency conflicts when running ``pipenv update --outdated``. diff --git a/news/2737.bugfix.rst b/news/2737.bugfix.rst deleted file mode 100644 index 5716c8dc..00000000 --- a/news/2737.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Non-ascii characters will now be handled correctly when parsed by pipenv's ``ToML`` parsers. diff --git a/news/2825.feature b/news/2825.feature deleted file mode 100644 index ba112a58..00000000 --- a/news/2825.feature +++ /dev/null @@ -1,2 +0,0 @@ -Added environment variable `PIPENV_PYUP_API_KEY` to add ability -to override the bundled pyup.io API key. diff --git a/news/2828.feature.rst b/news/2828.feature.rst deleted file mode 100644 index 688c47ee..00000000 --- a/news/2828.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added additional output to ``pipenv update --outdated`` to indicate that the operation succeded and all packages were already up to date. diff --git a/news/2848.bugfix.rst b/news/2848.bugfix.rst deleted file mode 100644 index f88c030d..00000000 --- a/news/2848.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Updated ``pipenv uninstall`` to respect the ``--skip-lock`` argument. diff --git a/news/2885.bugfix.rst b/news/2885.bugfix.rst deleted file mode 100644 index 72a7656c..00000000 --- a/news/2885.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug which caused uninstallation to sometimes fail to successfullly remove packages from ``Pipfiles`` with comments on preceding or following lines. diff --git a/news/2983.bugfix.rst b/news/2983.bugfix.rst deleted file mode 100644 index 87dcd71f..00000000 --- a/news/2983.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Pipenv will no longer fail when encountering python versions on Windows that have been uninstalled. diff --git a/news/2996.trivial b/news/2996.trivial deleted file mode 100644 index dbe1006a..00000000 --- a/news/2996.trivial +++ /dev/null @@ -1 +0,0 @@ -Set `PIPENV_ACTIVE=1` when running `pipenv run`. This is what `pipenv shell` already does. \ No newline at end of file diff --git a/news/3020.feature.rst b/news/3020.feature.rst deleted file mode 100644 index aebfab57..00000000 --- a/news/3020.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Updated ``crayons`` patch to enable colors on native powershell but swap native blue for magenta. diff --git a/news/3026.bugfix b/news/3026.bugfix deleted file mode 100644 index 6a194d1f..00000000 --- a/news/3026.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed unnecessary extras are added when translating markers diff --git a/news/3041.feature b/news/3041.feature deleted file mode 100644 index 8a87b5f6..00000000 --- a/news/3041.feature +++ /dev/null @@ -1 +0,0 @@ -Added support for ``--bare`` to ``pipenv clean``, and fixed ``pipenv sync --bare`` to actually reduce output. diff --git a/news/3047.bugfix.rst b/news/3047.bugfix.rst deleted file mode 100644 index 6c44bd42..00000000 --- a/news/3047.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a virtualenv creation issue which could cause new virtualenvs to inadvertently attempt to read and write to global site packages. diff --git a/news/3055.bugfix.rst b/news/3055.bugfix.rst deleted file mode 100644 index 7b2a0fa1..00000000 --- a/news/3055.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an issue with virtualenv path derivation which could cause errors, particularly for users on WSL bash. diff --git a/news/3061.vendor.rst b/news/3061.vendor.rst deleted file mode 100644 index fb815c42..00000000 --- a/news/3061.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. diff --git a/news/3069.trivial b/news/3069.trivial deleted file mode 100644 index 733a3cac..00000000 --- a/news/3069.trivial +++ /dev/null @@ -1 +0,0 @@ -- Fix PIPENV_VERBOSITY parsing logic diff --git a/news/3074.doc.rst b/news/3074.doc.rst deleted file mode 100644 index 85feefd5..00000000 --- a/news/3074.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Expanded development and testing documentation for contributors to get started. diff --git a/news/3088.bugfix.rst b/news/3088.bugfix.rst deleted file mode 100644 index fa57c42d..00000000 --- a/news/3088.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug which caused ``Unexpected EOF`` errors to be thrown when ``pip`` was waiting for input from users who had put login credentials in environment variables. diff --git a/news/3089.feature.rst b/news/3089.feature.rst deleted file mode 100644 index 47f280ee..00000000 --- a/news/3089.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added windows-compatible spinner via upgraded ``vistir`` dependency. diff --git a/news/3089.vendor.rst b/news/3089.vendor.rst deleted file mode 100644 index c57209f6..00000000 --- a/news/3089.vendor.rst +++ /dev/null @@ -1,11 +0,0 @@ -Updated vendored dependencies: - - ``certifi 2018.08.24 => 2018.10.15`` - - ``urllib3 1.23 => 1.24`` - - ``requests 2.19.1 => 2.20.0`` - - ``shellingham ``1.2.6 => 1.2.7`` - - ``tomlkit 0.4.4. => 0.4.6`` - - ``vistir 0.1.6 => 0.1.8`` - - ``pythonfinder 0.1.2 => 0.1.3`` - - ``requirementslib 1.1.9 => 1.1.10`` - - ``backports.functools_lru_cache 1.5.0 (new)`` - - ``cursor 1.2.0 (new)`` diff --git a/news/3090.bugfix.rst b/news/3090.bugfix.rst deleted file mode 100644 index af772c3d..00000000 --- a/news/3090.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug in ``requirementslib`` which prevented successful installation from mercurial repositories. diff --git a/news/3094.bugfix.rst b/news/3094.bugfix.rst deleted file mode 100644 index e17b2e32..00000000 --- a/news/3094.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed random resource warnings when using pyenv or any other subprocess calls. diff --git a/news/3096.bugfix.rst b/news/3096.bugfix.rst deleted file mode 100644 index 657abbb2..00000000 --- a/news/3096.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -- Fixed a bug which sometimes prevented cloning and parsing ``mercurial`` requirements. diff --git a/news/3096.feature.rst b/news/3096.feature.rst deleted file mode 100644 index b483d545..00000000 --- a/news/3096.feature.rst +++ /dev/null @@ -1 +0,0 @@ -- Added support for python installations managed by ``asdf``. diff --git a/news/3096.vendor.rst b/news/3096.vendor.rst deleted file mode 100644 index ff2cc836..00000000 --- a/news/3096.vendor.rst +++ /dev/null @@ -1,6 +0,0 @@ -Updated vendored dependencies: - - ``requests 2.19.1 => 2.20.1`` - - ``tomlkit 0.4.46 => 0.5.2`` - - ``vistir 0.1.6 => 0.2.4`` - - ``pythonfinder 1.1.2 => 1.1.8`` - - ``requirementslib 1.1.10 => 1.3.0`` diff --git a/news/3099.bugfix.rst b/news/3099.bugfix.rst deleted file mode 100644 index 72a7656c..00000000 --- a/news/3099.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug which caused uninstallation to sometimes fail to successfullly remove packages from ``Pipfiles`` with comments on preceding or following lines. diff --git a/news/3102.bugfix.rst b/news/3102.bugfix.rst deleted file mode 100644 index 2224ac0b..00000000 --- a/news/3102.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3104.bugfix b/news/3104.bugfix deleted file mode 100644 index 1915494d..00000000 --- a/news/3104.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the path casing issue that makes `pipenv clean` fail on Windows diff --git a/news/3106.bugfix.rst b/news/3106.bugfix.rst deleted file mode 100644 index 70c0917f..00000000 --- a/news/3106.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Pipenv will avoid leaving build artifacts in the current working directory. diff --git a/news/3109.bugfix.rst b/news/3109.bugfix.rst deleted file mode 100644 index af5718b6..00000000 --- a/news/3109.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures. diff --git a/news/3113.bugfix.rst b/news/3113.bugfix.rst deleted file mode 100644 index 75ee6de4..00000000 --- a/news/3113.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an issue which caused ``pipenv clean`` to sometimes clean packages from the base ``site-packages`` folder or fail entirely. diff --git a/news/3114.bugfix.rst b/news/3114.bugfix.rst deleted file mode 100644 index 2224ac0b..00000000 --- a/news/3114.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3117.bugfix.rst b/news/3117.bugfix.rst deleted file mode 100644 index 2224ac0b..00000000 --- a/news/3117.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. diff --git a/news/3121.bugfix.rst b/news/3121.bugfix.rst deleted file mode 100644 index fb815c42..00000000 --- a/news/3121.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. diff --git a/news/3121.vendor.rst b/news/3121.vendor.rst deleted file mode 100644 index fb815c42..00000000 --- a/news/3121.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. diff --git a/news/3131.bugfix.rst b/news/3131.bugfix.rst deleted file mode 100644 index 6eaa4bdd..00000000 --- a/news/3131.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Added additional logic for ignoring and replacing non-ascii characters when formatting console output on non-UTF-8 systems. diff --git a/news/3134.bugfix.rst b/news/3134.bugfix.rst deleted file mode 100644 index 9193321a..00000000 --- a/news/3134.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix virtual environment discovery when `PIPENV_VENV_IN_PROJECT` is set, but the in-project `.venv` is a file. diff --git a/news/3145.bugfix.rst b/news/3145.bugfix.rst deleted file mode 100644 index e0ed5095..00000000 --- a/news/3145.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Hashes for remote and local non-PyPI artifacts will now be included in ``Pipfile.lock`` during resolution. diff --git a/news/3151.bugfix b/news/3151.bugfix deleted file mode 100644 index 96f14c10..00000000 --- a/news/3151.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix project path hashing logic in purpose to prevent collisions of virtual environments. diff --git a/news/3154.trivial.rst b/news/3154.trivial.rst deleted file mode 100644 index 0c3277a6..00000000 --- a/news/3154.trivial.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an unreleased bug with determining project virtualenv paths across python versions. diff --git a/news/3158.bugfix.rst b/news/3158.bugfix.rst deleted file mode 100644 index 661fb09e..00000000 --- a/news/3158.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix package installation when the virtual environment path contains parentheses. diff --git a/news/3158.feature.rst b/news/3158.feature.rst deleted file mode 100644 index 08113e54..00000000 --- a/news/3158.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Improved runtime performance of no-op commands such as ``pipenv --venv`` by around 2/3. diff --git a/news/3164.bugfix.rst b/news/3164.bugfix.rst deleted file mode 100644 index 7b3fb1ac..00000000 --- a/news/3164.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Azure Pipelines YAML files are updated to use the latest syntax and product name. diff --git a/news/3170.feature b/news/3170.feature deleted file mode 100644 index eae5cbac..00000000 --- a/news/3170.feature +++ /dev/null @@ -1 +0,0 @@ -Do not show error but success for running ``pipenv uninstall --all`` in a fresh virtual environment. diff --git a/news/3179.trivial b/news/3179.trivial deleted file mode 100644 index 4b9059f7..00000000 --- a/news/3179.trivial +++ /dev/null @@ -1 +0,0 @@ -Better output of pipenv uninstall and clean diff --git a/news/3183.bugfix.rst b/news/3183.bugfix.rst deleted file mode 100644 index 19e1d54a..00000000 --- a/news/3183.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed new spinner success message to write only one success message during resolution. diff --git a/news/3185.bugfix.rst b/news/3185.bugfix.rst deleted file mode 100644 index b6ffee2b..00000000 --- a/news/3185.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Pipenv will now correctly respect the ``--pre`` option when used with ``pipenv install``. diff --git a/news/3191.vendor.rst b/news/3191.vendor.rst deleted file mode 100644 index 3806b68f..00000000 --- a/news/3191.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Switch to ``tomlkit`` for parsing and writing. Drop ``prettytoml`` and ``contoml`` from vendors. diff --git a/news/3196.vendor.rst b/news/3196.vendor.rst deleted file mode 100644 index 19351e2e..00000000 --- a/news/3196.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Updated ``requirementslib`` to aid in resolution of local and remote archives. diff --git a/news/3199.trivial b/news/3199.trivial deleted file mode 100644 index 1fcca7de..00000000 --- a/news/3199.trivial +++ /dev/null @@ -1 +0,0 @@ -Remove blank lines for pipenv graph diff --git a/news/3201.bugfix b/news/3201.bugfix deleted file mode 100644 index 01dba4f6..00000000 --- a/news/3201.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where exception is raised when run pipenv graph in a project without created virtualenv diff --git a/news/3216.bugfix.rst b/news/3216.bugfix.rst deleted file mode 100644 index 1d600bb8..00000000 --- a/news/3216.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -When sources are missing names, names will now be derived from the supplied URL. diff --git a/news/3217.feature.rst b/news/3217.feature.rst deleted file mode 100644 index f868fd95..00000000 --- a/news/3217.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Improved asynchronous installation and error handling via queued subprocess paralleization. diff --git a/pipenv/__version__.py b/pipenv/__version__.py index b2df6ad1..72fa8064 100644 --- a/pipenv/__version__.py +++ b/pipenv/__version__.py @@ -2,4 +2,4 @@ # // ) ) / / // ) ) //___) ) // ) ) || / / # //___/ / / / //___/ / // // / / || / / # // / / // ((____ // / / ||/ / -__version__ = "2018.10.14.dev0" +__version__ = "2018.11.14" From 76247ea3c3ef82e17e710f4fb77da45f3f7d5851 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 14 Nov 2018 19:31:58 -0500 Subject: [PATCH 197/218] pre-bump Signed-off-by: Dan Ryan --- pipenv/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/__version__.py b/pipenv/__version__.py index 72fa8064..802c3b9c 100644 --- a/pipenv/__version__.py +++ b/pipenv/__version__.py @@ -2,4 +2,4 @@ # // ) ) / / // ) ) //___) ) // ) ) || / / # //___/ / / / //___/ / // // / / || / / # // / / // ((____ // / / ||/ / -__version__ = "2018.11.14" +__version__ = "2018.11.15.dev0" From 895c4049b9453134219ff4385829eabd32858beb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 00:47:51 -0500 Subject: [PATCH 198/218] Fix encoding issues and python discovery - Add spiner symbols to unicode translation map - Update pythonfinder to master (ignore vendor update for now) - Fixes #3223 - Fixes #3224 Signed-off-by: Dan Ryan --- news/3223.bugfix.rst | 1 + news/3224.bugfix.rst | 1 + pipenv/_compat.py | 4 +++- pipenv/environments.py | 7 ++++--- pipenv/vendor/pythonfinder/models/python.py | 4 ++-- 5 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 news/3223.bugfix.rst create mode 100644 news/3224.bugfix.rst diff --git a/news/3223.bugfix.rst b/news/3223.bugfix.rst new file mode 100644 index 00000000..44f4a83e --- /dev/null +++ b/news/3223.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue with attempting to render unicode output in non-unicode locales. diff --git a/news/3224.bugfix.rst b/news/3224.bugfix.rst new file mode 100644 index 00000000..e6eed912 --- /dev/null +++ b/news/3224.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which could cause failures to occur when parsing python entries from global pyenv version files. diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 1dc52d5f..35ff2188 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -365,7 +365,9 @@ OUT_ENCODING, ERR_ENCODING = force_encoding() UNICODE_TO_ASCII_TRANSLATION_MAP = { 8230: u"...", - 8211: u"-" + 8211: u"-", + 10004: u"x", + 10008: u"Ok" } diff --git a/pipenv/environments.py b/pipenv/environments.py index c7d948df..066c296d 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -3,7 +3,8 @@ import os import sys from appdirs import user_cache_dir -from .vendor.vistir.misc import fs_str, to_text +from .vendor.vistir.misc import fs_str +from ._compat import fix_utf8 # HACK: avoid resolver.py uses the wrong byte code files. @@ -275,6 +276,6 @@ def is_quiet(threshold=-1): return PIPENV_VERBOSITY <= threshold -PIPENV_SPINNER_FAIL_TEXT = fs_str(to_text(u"✘ {0}")) if not PIPENV_HIDE_EMOJIS else ("{0}") +PIPENV_SPINNER_FAIL_TEXT = fix_utf8(u"✘ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") -PIPENV_SPINNER_OK_TEXT = fs_str(to_text(u"✔ {0}")) if not PIPENV_HIDE_EMOJIS else ("{0}") +PIPENV_SPINNER_OK_TEXT = fix_utf8(u"✔ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index db7d011e..eac856c5 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -67,9 +67,9 @@ class PythonFinder(BaseFinder, BasePath): ] versions = {v.name: v for v in version_paths} if self.is_pyenv: - version_order = [versions[v] for v in parse_pyenv_version_order()] + version_order = [versions[v] for v in parse_pyenv_version_order() if v in versions] elif self.is_asdf: - version_order = [versions[v] for v in parse_asdf_version_order()] + version_order = [versions[v] for v in parse_asdf_version_order() if v in versions] for version in version_order: version_paths.remove(version) if version_order: From ec604cffc33c1c35ee5e0b6950d56c8145a819a3 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 00:49:15 -0500 Subject: [PATCH 199/218] Add distributed testing for azure Signed-off-by: Dan Ryan --- .azure-pipelines/steps/DistributeTests.ps1 | 43 ++++++++++++++++++++++ .azure-pipelines/steps/run-tests.yml | 7 +++- 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 .azure-pipelines/steps/DistributeTests.ps1 diff --git a/.azure-pipelines/steps/DistributeTests.ps1 b/.azure-pipelines/steps/DistributeTests.ps1 new file mode 100644 index 00000000..d9fce4a3 --- /dev/null +++ b/.azure-pipelines/steps/DistributeTests.ps1 @@ -0,0 +1,43 @@ +<# +.SYNOPSIS + Distribute the tests in VSTS pipeline across multiple agents +.DESCRIPTION + This script slices tests files across multiple agents for faster execution. + We search for specific type of file structure (in this example test*), and slice them according to agent number + If we encounter multiple files [file1..file10] and if we have 2 agents, agent1 executes tests odd number of files while agent2 executes even number of files + For detalied slicing info: https://docs.microsoft.com/en-us/vsts/pipelines/test/parallel-testing-any-test-runner + We use JUnit style test results to publish the test reports. +#> + +$tests = Get-ChildItem ..\..\tests\unit,..\..\tests\integration -Filter "test*" # search for test files with specific pattern. +$totalAgents = [int]$Env:SYSTEM_TOTALJOBSINPHASE # standard VSTS variables available using parallel execution; total number of parallel jobs running +$agentNumber = [int]$Env:SYSTEM_JOBPOSITIONINPHASE # current job position +$testCount = $tests.Count + +# below conditions are used if parallel pipeline is not used. i.e. pipeline is running with single agent (no parallel configuration) +if ($totalAgents -eq 0) { + $totalAgents = 1 +} +if (!$agentNumber -or $agentNumber -eq 0) { + $agentNumber = 1 +} + +Write-Host "Total agents: $totalAgents" +Write-Host "Agent number: $agentNumber" +Write-Host "Total tests: $testCount" + +$testsToRun= @() + +# slice test files to make sure each agent gets unique test file to execute +For ($i=$agentNumber; $i -le $testCount;) { + $file = $tests[$i-1] + $testsToRun = $testsToRun + $file + Write-Host "Added $file" + $i = $i + $totalAgents + } + +# join all test files seperated by space. pytest runs multiple test files in following format pytest test1.py test2.py test3.py +$testFiles = $testsToRun -Join " " +Write-Host "Test files $testFiles" +# write these files into variable so that we can run them using pytest in subsequent task. +Write-Host "##vso[task.setvariable variable=pytestfiles;]$testFiles" diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index a7b99a13..6515f604 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,4 +1,7 @@ steps: +- powershell: ./DistributeTests.ps1 + displayName: 'PowerShell Script to distribute tests' + - powershell: | # Fix Git SSL errors pip install certifi @@ -11,8 +14,8 @@ steps: Write-Host "##vso[task.setvariable variable=TEMP]T:\" $env:TEMP='T:\' Write-Host "##vso[task.setvariable variable=TMP]T:\" - $env:TEMP='T:\' - D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests + $env:TMP='T:\' + D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml "$env:pytestfiles" displayName: Run integration tests - task: PublishTestResults@2 From 061869f22e78d3adc286f075976ae27dff52f55d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 00:56:21 -0500 Subject: [PATCH 200/218] Don't disable --user when --system is passed - Fixes #3222 Signed-off-by: Dan Ryan --- news/3222.bugfix.rst | 1 + pipenv/core.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 news/3222.bugfix.rst diff --git a/news/3222.bugfix.rst b/news/3222.bugfix.rst new file mode 100644 index 00000000..2f71f552 --- /dev/null +++ b/news/3222.bugfix.rst @@ -0,0 +1 @@ +Pipenv will no longer disable user-mode installation when the ``--system`` flag is passed in. diff --git a/pipenv/core.py b/pipenv/core.py index f815e5de..8c48d7fa 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -725,7 +725,8 @@ def batch_install(deps_list, procs, failed_deps_queue, extra_indexes = indexes[1:] with vistir.contextmanagers.temp_environ(): - os.environ["PIP_USER"] = vistir.compat.fs_str("0") + if not allow_global: + os.environ["PIP_USER"] = vistir.compat.fs_str("0") c = pip_install( dep, ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), @@ -1899,7 +1900,8 @@ def do_install( ) # pip install: with vistir.contextmanagers.temp_environ(), create_spinner("Installing...") as sp: - os.environ["PIP_USER"] = vistir.compat.fs_str("0") + if not system: + os.environ["PIP_USER"] = vistir.compat.fs_str("0") try: pkg_requirement = Requirement.from_line(pkg_line) except ValueError as e: From e94495ec8da8d0265fa50757aa8a3b3d7cc4e348 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:01:33 -0500 Subject: [PATCH 201/218] Fix circular import Signed-off-by: Dan Ryan --- pipenv/_compat.py | 6 +++--- pipenv/cli/command.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 35ff2188..09f65669 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -14,7 +14,6 @@ import sys import warnings import vistir from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp -from .utils import logging, rmtree try: from tempfile import _infer_return_type @@ -55,6 +54,7 @@ except ImportError: class finalize(object): def __init__(self, *args, **kwargs): + from .utils import logging logging.warn("weakref.finalize unavailable, not cleaning...") def detach(self): @@ -100,7 +100,7 @@ class TemporaryDirectory(object): @classmethod def _cleanup(cls, name, warn_message): - rmtree(name) + vistir.path.rmtree(name) warnings.warn(warn_message, ResourceWarning) def __repr__(self): @@ -114,7 +114,7 @@ class TemporaryDirectory(object): def cleanup(self): if self._finalizer.detach(): - rmtree(self.name) + vistir.path.rmtree(self.name) def _sanitize_params(prefix, suffix, dir): diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 14120e35..5a39fb02 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -15,7 +15,6 @@ import click_completion from click_didyoumean import DYMCommandCollection -from .. import environments from ..__version__ import __version__ from .options import ( CONTEXT_SETTINGS, PipenvGroup, code_option, common_options, deploy_option, @@ -115,6 +114,7 @@ def cli( return 1 if envs: echo("The following environment variables can be set, to do various things:\n") + from .. import environments for key in environments.__dict__: if key.startswith("PIPENV"): echo(" - {0}".format(crayons.normal(key, bold=True))) @@ -161,7 +161,8 @@ def cli( # --rm was passed… elif rm: # Abort if --system (or running in a virtualenv). - if environments.PIPENV_USE_SYSTEM: + from .environments import PIPENV_USE_SYSTEM + if PIPENV_USE_SYSTEM: echo( crayons.red( "You are attempting to remove a virtualenv that " From 04bb9ada05241e84a28aa0a3ee9cca36fdac8a84 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:01:33 -0500 Subject: [PATCH 202/218] Fix circular import Signed-off-by: Dan Ryan --- pipenv/_compat.py | 6 +++--- pipenv/cli/command.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 35ff2188..09f65669 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -14,7 +14,6 @@ import sys import warnings import vistir from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp -from .utils import logging, rmtree try: from tempfile import _infer_return_type @@ -55,6 +54,7 @@ except ImportError: class finalize(object): def __init__(self, *args, **kwargs): + from .utils import logging logging.warn("weakref.finalize unavailable, not cleaning...") def detach(self): @@ -100,7 +100,7 @@ class TemporaryDirectory(object): @classmethod def _cleanup(cls, name, warn_message): - rmtree(name) + vistir.path.rmtree(name) warnings.warn(warn_message, ResourceWarning) def __repr__(self): @@ -114,7 +114,7 @@ class TemporaryDirectory(object): def cleanup(self): if self._finalizer.detach(): - rmtree(self.name) + vistir.path.rmtree(self.name) def _sanitize_params(prefix, suffix, dir): diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 14120e35..5a39fb02 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -15,7 +15,6 @@ import click_completion from click_didyoumean import DYMCommandCollection -from .. import environments from ..__version__ import __version__ from .options import ( CONTEXT_SETTINGS, PipenvGroup, code_option, common_options, deploy_option, @@ -115,6 +114,7 @@ def cli( return 1 if envs: echo("The following environment variables can be set, to do various things:\n") + from .. import environments for key in environments.__dict__: if key.startswith("PIPENV"): echo(" - {0}".format(crayons.normal(key, bold=True))) @@ -161,7 +161,8 @@ def cli( # --rm was passed… elif rm: # Abort if --system (or running in a virtualenv). - if environments.PIPENV_USE_SYSTEM: + from .environments import PIPENV_USE_SYSTEM + if PIPENV_USE_SYSTEM: echo( crayons.red( "You are attempting to remove a virtualenv that " From ffe1403809d25104d8216d9ae0c058167c58ca4b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:23:08 -0500 Subject: [PATCH 203/218] Re-try dest distribution Signed-off-by: Dan Ryan --- .azure-pipelines/steps/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index 6515f604..fa45f558 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,5 +1,5 @@ steps: -- powershell: ./DistributeTests.ps1 +- powershell: .\azure-pipelines\steps\DistributeTests.ps1 displayName: 'PowerShell Script to distribute tests' - powershell: | From 6c8af23ec05fb8281f5d7ff0826003e0155b372b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:23:08 -0500 Subject: [PATCH 204/218] Re-try dest distribution Signed-off-by: Dan Ryan --- .azure-pipelines/steps/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index 6515f604..fa45f558 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,5 +1,5 @@ steps: -- powershell: ./DistributeTests.ps1 +- powershell: .\azure-pipelines\steps\DistributeTests.ps1 displayName: 'PowerShell Script to distribute tests' - powershell: | From ef235d8006ca5b898e119b7cbe8681440f9cce5e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:24:49 -0500 Subject: [PATCH 205/218] Fix import Signed-off-by: Dan Ryan --- pipenv/cli/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 5a39fb02..16778ede 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -161,7 +161,7 @@ def cli( # --rm was passed… elif rm: # Abort if --system (or running in a virtualenv). - from .environments import PIPENV_USE_SYSTEM + from ..environments import PIPENV_USE_SYSTEM if PIPENV_USE_SYSTEM: echo( crayons.red( From 5525a87c4b8ad90abfc4646aec38d56817170a42 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:28:03 -0500 Subject: [PATCH 206/218] fix run path for test runner script Signed-off-by: Dan Ryan --- .azure-pipelines/steps/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index fa45f558..cbddf0ee 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,5 +1,5 @@ steps: -- powershell: .\azure-pipelines\steps\DistributeTests.ps1 +- powershell: .\.azure-pipelines\steps\DistributeTests.ps1 displayName: 'PowerShell Script to distribute tests' - powershell: | From addecb10f91879a272ef9420929d8b82fc75594c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:28:03 -0500 Subject: [PATCH 207/218] fix run path for test runner script Signed-off-by: Dan Ryan --- .azure-pipelines/steps/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index fa45f558..cbddf0ee 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,5 +1,5 @@ steps: -- powershell: .\azure-pipelines\steps\DistributeTests.ps1 +- powershell: .\.azure-pipelines\steps\DistributeTests.ps1 displayName: 'PowerShell Script to distribute tests' - powershell: | From 0af52248481272a81da1cc71c70a89729ef976e2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:44:15 -0500 Subject: [PATCH 208/218] dont distribute tests, it doesn't work Signed-off-by: Dan Ryan --- .azure-pipelines/steps/DistributeTests.ps1 | 43 ---------------------- .azure-pipelines/steps/run-tests.yml | 5 +-- pipenv/cli/command.py | 2 +- 3 files changed, 2 insertions(+), 48 deletions(-) delete mode 100644 .azure-pipelines/steps/DistributeTests.ps1 diff --git a/.azure-pipelines/steps/DistributeTests.ps1 b/.azure-pipelines/steps/DistributeTests.ps1 deleted file mode 100644 index d9fce4a3..00000000 --- a/.azure-pipelines/steps/DistributeTests.ps1 +++ /dev/null @@ -1,43 +0,0 @@ -<# -.SYNOPSIS - Distribute the tests in VSTS pipeline across multiple agents -.DESCRIPTION - This script slices tests files across multiple agents for faster execution. - We search for specific type of file structure (in this example test*), and slice them according to agent number - If we encounter multiple files [file1..file10] and if we have 2 agents, agent1 executes tests odd number of files while agent2 executes even number of files - For detalied slicing info: https://docs.microsoft.com/en-us/vsts/pipelines/test/parallel-testing-any-test-runner - We use JUnit style test results to publish the test reports. -#> - -$tests = Get-ChildItem ..\..\tests\unit,..\..\tests\integration -Filter "test*" # search for test files with specific pattern. -$totalAgents = [int]$Env:SYSTEM_TOTALJOBSINPHASE # standard VSTS variables available using parallel execution; total number of parallel jobs running -$agentNumber = [int]$Env:SYSTEM_JOBPOSITIONINPHASE # current job position -$testCount = $tests.Count - -# below conditions are used if parallel pipeline is not used. i.e. pipeline is running with single agent (no parallel configuration) -if ($totalAgents -eq 0) { - $totalAgents = 1 -} -if (!$agentNumber -or $agentNumber -eq 0) { - $agentNumber = 1 -} - -Write-Host "Total agents: $totalAgents" -Write-Host "Agent number: $agentNumber" -Write-Host "Total tests: $testCount" - -$testsToRun= @() - -# slice test files to make sure each agent gets unique test file to execute -For ($i=$agentNumber; $i -le $testCount;) { - $file = $tests[$i-1] - $testsToRun = $testsToRun + $file - Write-Host "Added $file" - $i = $i + $totalAgents - } - -# join all test files seperated by space. pytest runs multiple test files in following format pytest test1.py test2.py test3.py -$testFiles = $testsToRun -Join " " -Write-Host "Test files $testFiles" -# write these files into variable so that we can run them using pytest in subsequent task. -Write-Host "##vso[task.setvariable variable=pytestfiles;]$testFiles" diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index cbddf0ee..7496c76f 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,7 +1,4 @@ steps: -- powershell: .\.azure-pipelines\steps\DistributeTests.ps1 - displayName: 'PowerShell Script to distribute tests' - - powershell: | # Fix Git SSL errors pip install certifi @@ -15,7 +12,7 @@ steps: $env:TEMP='T:\' Write-Host "##vso[task.setvariable variable=TMP]T:\" $env:TMP='T:\' - D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml "$env:pytestfiles" + D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests - task: PublishTestResults@2 diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 5a39fb02..16778ede 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -161,7 +161,7 @@ def cli( # --rm was passed… elif rm: # Abort if --system (or running in a virtualenv). - from .environments import PIPENV_USE_SYSTEM + from ..environments import PIPENV_USE_SYSTEM if PIPENV_USE_SYSTEM: echo( crayons.red( From 4359376e5a59ea0ef2c40ab8b04d4db91e06bd50 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:45:36 -0500 Subject: [PATCH 209/218] dont distribute tests, it doesn't work Signed-off-by: Dan Ryan --- .azure-pipelines/steps/DistributeTests.ps1 | 43 ---------------------- .azure-pipelines/steps/run-tests.yml | 5 +-- 2 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 .azure-pipelines/steps/DistributeTests.ps1 diff --git a/.azure-pipelines/steps/DistributeTests.ps1 b/.azure-pipelines/steps/DistributeTests.ps1 deleted file mode 100644 index d9fce4a3..00000000 --- a/.azure-pipelines/steps/DistributeTests.ps1 +++ /dev/null @@ -1,43 +0,0 @@ -<# -.SYNOPSIS - Distribute the tests in VSTS pipeline across multiple agents -.DESCRIPTION - This script slices tests files across multiple agents for faster execution. - We search for specific type of file structure (in this example test*), and slice them according to agent number - If we encounter multiple files [file1..file10] and if we have 2 agents, agent1 executes tests odd number of files while agent2 executes even number of files - For detalied slicing info: https://docs.microsoft.com/en-us/vsts/pipelines/test/parallel-testing-any-test-runner - We use JUnit style test results to publish the test reports. -#> - -$tests = Get-ChildItem ..\..\tests\unit,..\..\tests\integration -Filter "test*" # search for test files with specific pattern. -$totalAgents = [int]$Env:SYSTEM_TOTALJOBSINPHASE # standard VSTS variables available using parallel execution; total number of parallel jobs running -$agentNumber = [int]$Env:SYSTEM_JOBPOSITIONINPHASE # current job position -$testCount = $tests.Count - -# below conditions are used if parallel pipeline is not used. i.e. pipeline is running with single agent (no parallel configuration) -if ($totalAgents -eq 0) { - $totalAgents = 1 -} -if (!$agentNumber -or $agentNumber -eq 0) { - $agentNumber = 1 -} - -Write-Host "Total agents: $totalAgents" -Write-Host "Agent number: $agentNumber" -Write-Host "Total tests: $testCount" - -$testsToRun= @() - -# slice test files to make sure each agent gets unique test file to execute -For ($i=$agentNumber; $i -le $testCount;) { - $file = $tests[$i-1] - $testsToRun = $testsToRun + $file - Write-Host "Added $file" - $i = $i + $totalAgents - } - -# join all test files seperated by space. pytest runs multiple test files in following format pytest test1.py test2.py test3.py -$testFiles = $testsToRun -Join " " -Write-Host "Test files $testFiles" -# write these files into variable so that we can run them using pytest in subsequent task. -Write-Host "##vso[task.setvariable variable=pytestfiles;]$testFiles" diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index cbddf0ee..7496c76f 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,7 +1,4 @@ steps: -- powershell: .\.azure-pipelines\steps\DistributeTests.ps1 - displayName: 'PowerShell Script to distribute tests' - - powershell: | # Fix Git SSL errors pip install certifi @@ -15,7 +12,7 @@ steps: $env:TEMP='T:\' Write-Host "##vso[task.setvariable variable=TMP]T:\" $env:TMP='T:\' - D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml "$env:pytestfiles" + D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests - task: PublishTestResults@2 From cc03cd0cb46f84a554a233a7edc1aaa0947432bd Mon Sep 17 00:00:00 2001 From: kashewnuts Date: Thu, 15 Nov 2018 16:01:50 +0900 Subject: [PATCH 210/218] fix 'Release 2018.11.14' markup --- CHANGELOG.rst | 92 +++++++++++++++++++++++++-------------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3686ca85..45494be7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,59 +4,59 @@ Features & Improvements ----------------------- -- Improved exceptions and error handling on failures.`#1977 `_ -- Added persistent settings for all CLI flags via ``PIPENV_{FLAG_NAME}`` environment variables by enabling ``auto_envvar_prefix=PIPENV`` in click (implements PEEP-0002).`#2200 `_ -- Added improved messaging about available but skipped updates due to dependency conflicts when running ``pipenv update --outdated``.`#2411 `_ +- Improved exceptions and error handling on failures. `#1977 `_ +- Added persistent settings for all CLI flags via ``PIPENV_{FLAG_NAME}`` environment variables by enabling ``auto_envvar_prefix=PIPENV`` in click (implements PEEP-0002). `#2200 `_ +- Added improved messaging about available but skipped updates due to dependency conflicts when running ``pipenv update --outdated``. `#2411 `_ - Added environment variable `PIPENV_PYUP_API_KEY` to add ability - to override the bundled pyup.io API key.`#2825 `_ -- Added additional output to ``pipenv update --outdated`` to indicate that the operation succeded and all packages were already up to date.`#2828 `_ -- Updated ``crayons`` patch to enable colors on native powershell but swap native blue for magenta.`#3020 `_ -- Added support for ``--bare`` to ``pipenv clean``, and fixed ``pipenv sync --bare`` to actually reduce output.`#3041 `_ -- Added windows-compatible spinner via upgraded ``vistir`` dependency.`#3089 `_ -- - Added support for python installations managed by ``asdf``.`#3096 `_ -- Improved runtime performance of no-op commands such as ``pipenv --venv`` by around 2/3.`#3158 `_ -- Do not show error but success for running ``pipenv uninstall --all`` in a fresh virtual environment.`#3170 `_ -- Improved asynchronous installation and error handling via queued subprocess paralleization.`#3217 `_ + to override the bundled pyup.io API key. `#2825 `_ +- Added additional output to ``pipenv update --outdated`` to indicate that the operation succeded and all packages were already up to date. `#2828 `_ +- Updated ``crayons`` patch to enable colors on native powershell but swap native blue for magenta. `#3020 `_ +- Added support for ``--bare`` to ``pipenv clean``, and fixed ``pipenv sync --bare`` to actually reduce output. `#3041 `_ +- Added windows-compatible spinner via upgraded ``vistir`` dependency. `#3089 `_ +- - Added support for python installations managed by ``asdf``. `#3096 `_ +- Improved runtime performance of no-op commands such as ``pipenv --venv`` by around 2/3. `#3158 `_ +- Do not show error but success for running ``pipenv uninstall --all`` in a fresh virtual environment. `#3170 `_ +- Improved asynchronous installation and error handling via queued subprocess paralleization. `#3217 `_ Bug Fixes --------- -- Remote non-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``.`#2394 `_ -- Non-ascii characters will now be handled correctly when parsed by pipenv's ``ToML`` parsers.`#2737 `_ -- Updated ``pipenv uninstall`` to respect the ``--skip-lock`` argument.`#2848 `_ -- Fixed a bug which caused uninstallation to sometimes fail to successfullly remove packages from ``Pipfiles`` with comments on preceding or following lines.`#2885 `_, +- Remote non-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. `#2394 `_ +- Non-ascii characters will now be handled correctly when parsed by pipenv's ``ToML`` parsers. `#2737 `_ +- Updated ``pipenv uninstall`` to respect the ``--skip-lock`` argument. `#2848 `_ +- Fixed a bug which caused uninstallation to sometimes fail to successfullly remove packages from ``Pipfiles`` with comments on preceding or following lines. `#2885 `_, `#3099 `_ -- Pipenv will no longer fail when encountering python versions on Windows that have been uninstalled.`#2983 `_ -- Fixed unnecessary extras are added when translating markers`#3026 `_ -- Fixed a virtualenv creation issue which could cause new virtualenvs to inadvertently attempt to read and write to global site packages.`#3047 `_ -- Fixed an issue with virtualenv path derivation which could cause errors, particularly for users on WSL bash.`#3055 `_ -- Fixed a bug which caused ``Unexpected EOF`` errors to be thrown when ``pip`` was waiting for input from users who had put login credentials in environment variables.`#3088 `_ -- Fixed a bug in ``requirementslib`` which prevented successful installation from mercurial repositories.`#3090 `_ -- Fixed random resource warnings when using pyenv or any other subprocess calls.`#3094 `_ -- - Fixed a bug which sometimes prevented cloning and parsing ``mercurial`` requirements.`#3096 `_ -- Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors.`#3102 `_, +- Pipenv will no longer fail when encountering python versions on Windows that have been uninstalled. `#2983 `_ +- Fixed unnecessary extras are added when translating markers `#3026 `_ +- Fixed a virtualenv creation issue which could cause new virtualenvs to inadvertently attempt to read and write to global site packages. `#3047 `_ +- Fixed an issue with virtualenv path derivation which could cause errors, particularly for users on WSL bash. `#3055 `_ +- Fixed a bug which caused ``Unexpected EOF`` errors to be thrown when ``pip`` was waiting for input from users who had put login credentials in environment variables. `#3088 `_ +- Fixed a bug in ``requirementslib`` which prevented successful installation from mercurial repositories. `#3090 `_ +- Fixed random resource warnings when using pyenv or any other subprocess calls. `#3094 `_ +- - Fixed a bug which sometimes prevented cloning and parsing ``mercurial`` requirements. `#3096 `_ +- Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. `#3102 `_, `#3114 `_, `#3117 `_ -- Fix the path casing issue that makes `pipenv clean` fail on Windows`#3104 `_ -- Pipenv will avoid leaving build artifacts in the current working directory.`#3106 `_ -- Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures.`#3109 `_ -- Fixed an issue which caused ``pipenv clean`` to sometimes clean packages from the base ``site-packages`` folder or fail entirely.`#3113 `_ -- Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions.`#3121 `_ -- Added additional logic for ignoring and replacing non-ascii characters when formatting console output on non-UTF-8 systems.`#3131 `_ -- Fix virtual environment discovery when `PIPENV_VENV_IN_PROJECT` is set, but the in-project `.venv` is a file.`#3134 `_ -- Hashes for remote and local non-PyPI artifacts will now be included in ``Pipfile.lock`` during resolution.`#3145 `_ -- Fix project path hashing logic in purpose to prevent collisions of virtual environments.`#3151 `_ -- Fix package installation when the virtual environment path contains parentheses.`#3158 `_ -- Azure Pipelines YAML files are updated to use the latest syntax and product name.`#3164 `_ -- Fixed new spinner success message to write only one success message during resolution.`#3183 `_ -- Pipenv will now correctly respect the ``--pre`` option when used with ``pipenv install``.`#3185 `_ -- Fix a bug where exception is raised when run pipenv graph in a project without created virtualenv`#3201 `_ -- When sources are missing names, names will now be derived from the supplied URL.`#3216 `_ +- Fix the path casing issue that makes `pipenv clean` fail on Windows `#3104 `_ +- Pipenv will avoid leaving build artifacts in the current working directory. `#3106 `_ +- Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures. `#3109 `_ +- Fixed an issue which caused ``pipenv clean`` to sometimes clean packages from the base ``site-packages`` folder or fail entirely. `#3113 `_ +- Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. `#3121 `_ +- Added additional logic for ignoring and replacing non-ascii characters when formatting console output on non-UTF-8 systems. `#3131 `_ +- Fix virtual environment discovery when `PIPENV_VENV_IN_PROJECT` is set, but the in-project `.venv` is a file. `#3134 `_ +- Hashes for remote and local non-PyPI artifacts will now be included in ``Pipfile.lock`` during resolution. `#3145 `_ +- Fix project path hashing logic in purpose to prevent collisions of virtual environments. `#3151 `_ +- Fix package installation when the virtual environment path contains parentheses. `#3158 `_ +- Azure Pipelines YAML files are updated to use the latest syntax and product name. `#3164 `_ +- Fixed new spinner success message to write only one success message during resolution. `#3183 `_ +- Pipenv will now correctly respect the ``--pre`` option when used with ``pipenv install``. `#3185 `_ +- Fix a bug where exception is raised when run pipenv graph in a project without created virtualenv `#3201 `_ +- When sources are missing names, names will now be derived from the supplied URL. `#3216 `_ Vendored Libraries ------------------ -- Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions.`#3061 `_, +- Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. `#3061 `_, `#3121 `_ - Updated vendored dependencies: - ``certifi 2018.08.24 => 2018.10.15`` @@ -68,20 +68,20 @@ Vendored Libraries - ``pythonfinder 0.1.2 => 0.1.3`` - ``requirementslib 1.1.9 => 1.1.10`` - ``backports.functools_lru_cache 1.5.0 (new)`` - - ``cursor 1.2.0 (new)```#3089 `_ + - ``cursor 1.2.0 (new)`` `#3089 `_ - Updated vendored dependencies: - ``requests 2.19.1 => 2.20.1`` - ``tomlkit 0.4.46 => 0.5.2`` - ``vistir 0.1.6 => 0.2.4`` - ``pythonfinder 1.1.2 => 1.1.8`` - - ``requirementslib 1.1.10 => 1.3.0```#3096 `_ -- Switch to ``tomlkit`` for parsing and writing. Drop ``prettytoml`` and ``contoml`` from vendors.`#3191 `_ -- Updated ``requirementslib`` to aid in resolution of local and remote archives.`#3196 `_ + - ``requirementslib 1.1.10 => 1.3.0`` `#3096 `_ +- Switch to ``tomlkit`` for parsing and writing. Drop ``prettytoml`` and ``contoml`` from vendors. `#3191 `_ +- Updated ``requirementslib`` to aid in resolution of local and remote archives. `#3196 `_ Improved Documentation ---------------------- -- Expanded development and testing documentation for contributors to get started.`#3074 `_ +- Expanded development and testing documentation for contributors to get started. `#3074 `_ 2018.10.13 (2018-10-13) From 262293699e583f39e97f8e95e1215f4b257c528b Mon Sep 17 00:00:00 2001 From: Chris Miller Date: Fri, 16 Nov 2018 00:26:35 -0500 Subject: [PATCH 211/218] Fix race condition when installing 2+ editable non-VCS pkgs at once This regression was recently introduced, and only affects non-vcs packages. The effect of the race is that installations of multiple editable non-VCS sourced packages at once may cause some of them to be un-importable. The fix is to make editable package installs Blocking just like VCS installs are. --- pipenv/core.py | 2 +- tests/integration/test_install_twists.py | 53 ++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index f815e5de..f9ffeba3 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -731,7 +731,7 @@ def batch_install(deps_list, procs, failed_deps_queue, ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), allow_global=allow_global, no_deps=False if is_artifact else no_deps, - block=any([dep.is_vcs, blocking]), + block=any([dep.editable, dep.is_vcs, blocking]), index=index, requirements_dir=requirements_dir, pypi_mirror=pypi_mirror, diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 055c39b2..6b202d4d 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -8,6 +8,7 @@ from pipenv.utils import mkdir_p, temp_environ import pytest from flaky import flaky +import delegator @pytest.mark.extras @@ -333,3 +334,55 @@ six = {{path = "./artifacts/{}"}} c = p.pipenv("install") assert c.return_code == 0 assert "six" in p.lockfile["default"] + + +@pytest.mark.files +@pytest.mark.needs_internet +@pytest.mark.install +@pytest.mark.run +def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, tmpdir, testsroot): + """Test for a race condition that can occur when installing multiple 'editable' packages at + once, and which causes some of them to not be importable. + + This issue had been fixed for VCS packages already, but not local 'editable' packages. + + So this test locally installs packages from tarballs that have already been committed in + the local `pypi` dir to avoid using VCS packages. + """ + pkgs = { + "requests-2.19.1": "requests/requests-2.19.1.tar.gz", + "flask-0.12.2": "flask/flask-0.12.2.tar.gz", + "six-1.11.0": "six/six-1.11.0.tar.gz", + "jinja2-2.10": "jinja2/jinja2-2.10.tar.gz", + } + + pipfile_string=""" +[packages] +""" + # Unzip tarballs to known location, and update Pipfile template. + for pkg_name, file_name in pkgs.items(): + source_path = os.path.abspath(os.path.join(testsroot, "pypi", file_name)) + unzip_path = os.path.join(tmpdir, pkg_name) + + import tarfile + + with tarfile.open(source_path, "r:gz") as tgz: + tgz.extractall(path=tmpdir) + + pipfile_string += '"{0}" = {{path = "{1}", editable = true}}\n'.format(pkg_name, unzip_path) + + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(pipfile_string.strip()) + + c = p.pipenv('install') + assert c.return_code == 0 + + c = p.pipenv('run python -c "import requests"') + assert c.return_code == 0 + c = p.pipenv('run python -c "import flask"') + assert c.return_code == 0 + c = p.pipenv('run python -c "import six"') + assert c.return_code == 0 + c = p.pipenv('run python -c "import jinja2"') + assert c.return_code == 0 From a95ce8bb32efea8c73fa8b7e1c4290319444405a Mon Sep 17 00:00:00 2001 From: Chris Miller Date: Fri, 16 Nov 2018 08:14:21 -0500 Subject: [PATCH 212/218] Remove unnecessary test marks and imports. --- tests/integration/test_install_twists.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 6b202d4d..ee11a186 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -8,7 +8,6 @@ from pipenv.utils import mkdir_p, temp_environ import pytest from flaky import flaky -import delegator @pytest.mark.extras @@ -337,7 +336,6 @@ six = {{path = "./artifacts/{}"}} @pytest.mark.files -@pytest.mark.needs_internet @pytest.mark.install @pytest.mark.run def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, tmpdir, testsroot): @@ -351,9 +349,9 @@ def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, tmpdir """ pkgs = { "requests-2.19.1": "requests/requests-2.19.1.tar.gz", - "flask-0.12.2": "flask/flask-0.12.2.tar.gz", + "Flask-0.12.2": "flask/Flask-0.12.2.tar.gz", "six-1.11.0": "six/six-1.11.0.tar.gz", - "jinja2-2.10": "jinja2/jinja2-2.10.tar.gz", + "Jinja2-2.10": "jinja2/Jinja2-2.10.tar.gz", } pipfile_string=""" From db1eb1bacb9d8fe163f5f6c39304c532b262b5c6 Mon Sep 17 00:00:00 2001 From: Chris Miller Date: Fri, 16 Nov 2018 09:05:58 -0500 Subject: [PATCH 213/218] Use correct Path module to get tests to work. --- tests/integration/test_install_twists.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index ee11a186..cef6c34d 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -360,14 +360,14 @@ def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, tmpdir # Unzip tarballs to known location, and update Pipfile template. for pkg_name, file_name in pkgs.items(): source_path = os.path.abspath(os.path.join(testsroot, "pypi", file_name)) - unzip_path = os.path.join(tmpdir, pkg_name) + unzip_path = tmpdir.join(pkg_name) import tarfile with tarfile.open(source_path, "r:gz") as tgz: - tgz.extractall(path=tmpdir) + tgz.extractall(path=tmpdir.strpath) - pipfile_string += '"{0}" = {{path = "{1}", editable = true}}\n'.format(pkg_name, unzip_path) + pipfile_string += '"{0}" = {{path = "{1}", editable = true}}\n'.format(pkg_name, unzip_path.strpath) with PipenvInstance(pypi=pypi, chdir=True) as p: with open(p.pipfile_path, 'w') as f: From 65e49f1bffcdd64db697c911c7723df2f6e794f5 Mon Sep 17 00:00:00 2001 From: Chris Miller Date: Fri, 16 Nov 2018 09:45:59 -0500 Subject: [PATCH 214/218] Print generated Pipfile during test to help debug failures. --- tests/integration/test_install_twists.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index cef6c34d..95dc058a 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -370,6 +370,7 @@ def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, tmpdir pipfile_string += '"{0}" = {{path = "{1}", editable = true}}\n'.format(pkg_name, unzip_path.strpath) with PipenvInstance(pypi=pypi, chdir=True) as p: + print(pipfile_string) with open(p.pipfile_path, 'w') as f: f.write(pipfile_string.strip()) From 70f6bec85e65d7b4f02a5c0299fa52b3d2cf75d2 Mon Sep 17 00:00:00 2001 From: Chris Miller Date: Fri, 16 Nov 2018 11:50:25 -0500 Subject: [PATCH 215/218] Use cross platform Path compatibility module in test. --- tests/integration/test_install_twists.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 95dc058a..92dedca8 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -359,15 +359,15 @@ def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, tmpdir """ # Unzip tarballs to known location, and update Pipfile template. for pkg_name, file_name in pkgs.items(): - source_path = os.path.abspath(os.path.join(testsroot, "pypi", file_name)) - unzip_path = tmpdir.join(pkg_name) + source_path = str(Path(testsroot, "pypi", file_name)) + unzip_path = str(Path(tmpdir.strpath, pkg_name)) import tarfile with tarfile.open(source_path, "r:gz") as tgz: tgz.extractall(path=tmpdir.strpath) - pipfile_string += '"{0}" = {{path = "{1}", editable = true}}\n'.format(pkg_name, unzip_path.strpath) + pipfile_string += "'{0}' = {{path = '{1}', editable = true}}\n".format(pkg_name, unzip_path) with PipenvInstance(pypi=pypi, chdir=True) as p: print(pipfile_string) From 905c8c062a872a30a1defb5c1ef97c9fd58974da Mon Sep 17 00:00:00 2001 From: frostming Date: Fri, 16 Nov 2018 10:51:18 +0800 Subject: [PATCH 216/218] Derive missing values of source from existing fields Signed-off-by: frostming --- pipenv/project.py | 22 +++++++++++++--------- tests/integration/test_lock.py | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 590fafe5..0067348c 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -727,6 +727,18 @@ class Project(object): data[u"requires"] = {"python_version": version[: len("2.7")]} self.write_toml(data) + @classmethod + def populate_source(cls, source): + """Derive missing values of source from the existing fields.""" + # Only URL pararemter is mandatory, let the KeyError be thrown. + if "name" not in source: + source["name"] = get_url_name(source["url"]) + if "verify_ssl" not in source: + source["verify_ssl"] = "https://" in source["url"] + if not isinstance(source["verify_ssl"], bool): + source["verify_ssl"] = source["verify_ssl"].lower() == "true" + return source + def get_or_create_lockfile(self): from pipenv.vendor.requirementslib.models.lockfile import Lockfile as Req_Lockfile lockfile = None @@ -747,15 +759,7 @@ class Project(object): elif not isinstance(sources, list): sources = [sources,] lockfile_dict["_meta"]["sources"] = [ - { - "name": s.get("name", get_url_name(s.get("url"))), - "url": s["url"], - "verify_ssl": ( - s["verify_ssl"] if isinstance(s["verify_ssl"], bool) else ( - True if s["verify_ssl"].lower() == "true" else False - ) - ) - } for s in sources + self.populate_source(s) for s in sources ] _created_lockfile = Req_Lockfile.from_data( path=self.lockfile_location, data=lockfile_dict, meta_from_project=False diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 2b520808..f2f0ac76 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -486,3 +486,20 @@ def test_lockfile_with_empty_dict(PipenvInstance): assert c.return_code == 0 assert 'Pipfile.lock is corrupted' in c.err assert p.lockfile['_meta'] + + +@pytest.mark.lock +@pytest.mark.install +def test_lock_with_incomplete_source(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[[source]] +url = "https://test.pypi.org/simple" + +[packages] +requests = "*" + """) + c = p.pipenv('install') + assert c.return_code == 0 + assert p.lockfile['_meta']['sources'] From cba5a1efc62530aff87558cf0c3aeac7c9896e4f Mon Sep 17 00:00:00 2001 From: Chris Miller Date: Sat, 17 Nov 2018 12:52:12 -0500 Subject: [PATCH 217/218] Remove debug print from test. --- tests/integration/test_install_twists.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 92dedca8..2ad12691 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -370,7 +370,6 @@ def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, tmpdir pipfile_string += "'{0}' = {{path = '{1}', editable = true}}\n".format(pkg_name, unzip_path) with PipenvInstance(pypi=pypi, chdir=True) as p: - print(pipfile_string) with open(p.pipfile_path, 'w') as f: f.write(pipfile_string.strip()) From 12b66e1e4ab3dc00468e8e6a4da1e5786f64ee4a Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sun, 18 Nov 2018 00:06:03 +0800 Subject: [PATCH 218/218] update manual Signed-off-by: Frost Ming --- docs/advanced.rst | 7 +- docs/basics.rst | 16 +- pipenv/cli/command.py | 2 +- pipenv/pipenv.1 | 2873 ++++++++++++++++++++++++++++++++++++++--- tasks/release.py | 11 + 5 files changed, 2728 insertions(+), 181 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index 3ac323e6..236f8c12 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -353,12 +353,12 @@ To prevent pipenv from loading the ``.env`` file, set the ``PIPENV_DONT_LOAD_ENV ☤ Custom Script Shortcuts ------------------------- -Pipenv supports creating custom shortcuts in the (optional) ``[scripts]`` section of your Pipfile. +Pipenv supports creating custom shortcuts in the (optional) ``[scripts]`` section of your Pipfile. You can then run ``pipenv run `` in your terminal to run the command in the -context of your pipenv virtual environment even if you have not activated the pipenv shell first. +context of your pipenv virtual environment even if you have not activated the pipenv shell first. -For example, in your Pipfile:: +For example, in your Pipfile:: [scripts] printspam = "python -c \"print('I am a silly example, no one would need to do this')\"" @@ -397,6 +397,7 @@ Pipenv supports the usage of environment variables in values. For example:: Environment variables may be specified as ``${MY_ENVAR}`` or ``$MY_ENVAR``. On Windows, ``%MY_ENVAR%`` is supported in addition to ``${MY_ENVAR}`` or ``$MY_ENVAR``. +.. _configuration-with-environment-variables: ☤ Configuration With Environment Variables ------------------------------------------ diff --git a/docs/basics.rst b/docs/basics.rst index 00f4c7d9..4ba6116d 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -182,12 +182,12 @@ in your ``Pipfile.lock`` for now, run ``pipenv lock --keep-outdated``. Make sur ☤ Specifying Versions of a Package ---------------------------------- -You can specify versions of a package using the `Semantic Versioning scheme `_ -(i.e. ``major.minor.micro``). +You can specify versions of a package using the `Semantic Versioning scheme `_ +(i.e. ``major.minor.micro``). For example, to install requests you can use: :: - $ pipenv install requests~=1.2 # equivalent to requests~=1.2.0 + $ pipenv install requests~=1.2 # equivalent to requests~=1.2.0 Pipenv will install version ``1.2`` and any minor update, but not ``2.0``. @@ -201,11 +201,11 @@ To make inclusive or exclusive version comparisons you can use: :: $ pipenv install "requests>=1.4" # will install a version equal or larger than 1.4.0 $ pipenv install "requests<=2.13" # will install a version equal or lower than 2.13.0 - $ pipenv install "requests>2.19" # will install 2.19.1 but not 2.19.0 + $ pipenv install "requests>2.19" # will install 2.19.1 but not 2.19.0 -.. note:: The use of ``" "`` around the package and version specification is highly recommended +.. note:: The use of ``" "`` around the package and version specification is highly recommended to avoid issues with `Input and output redirection `_ - in Unix-based operating systems. + in Unix-based operating systems. The use of ``~=`` is preferred over the ``==`` identifier as the former prevents pipenv from updating the packages: :: @@ -398,10 +398,8 @@ environment into production. You can use ``pipenv lock`` to compile your depende your development environment and deploy the compiled ``Pipfile.lock`` to all of your production environments for reproducible builds. -.. note: +.. note:: If you'd like a ``requirements.txt`` output of the lockfile, run ``$ pipenv lock -r``. This will include all hashes, however (which is great!). To get a ``requirements.txt`` without hashes, use ``$ pipenv run pip freeze``. - -.. _configuration-with-environment-variables:https://docs.pipenv.org/advanced/#configuration-with-environment-variables diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 16778ede..85f0b65a 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -106,7 +106,7 @@ def cli( if man: if system_which("man"): - path = os.sep.join([os.path.dirname(__file__), "pipenv.1"]) + path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "pipenv.1") os.execle(system_which("man"), "man", path, os.environ) return 0 else: diff --git a/pipenv/pipenv.1 b/pipenv/pipenv.1 index 62808744..0876d34f 100644 --- a/pipenv/pipenv.1 +++ b/pipenv/pipenv.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "PIPENV" "1" "Oct 07, 2017" "8.2.7" "pipenv" +.TH "PIPENV" "1" "Nov 18, 2018" "2018.11.15.dev0" "pipenv" .SH NAME pipenv \- pipenv Documentation . @@ -38,39 +38,58 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .ce 0 .sp .sp -\fBPipenv\fP —\ the officially recommended Python packaging tool from \fI\%Python.org\fP, free (as in freedom). +\fBPipenv\fP is a tool that aims to bring the best of all packaging worlds (bundler, composer, npm, cargo, yarn, etc.) to the Python world. \fIWindows is a first\-class citizen, in our world.\fP .sp -Pipenv is a tool that aims to bring the best of all packaging worlds (bundler, composer, npm, cargo, yarn, etc.) to the Python world. \fIWindows is a first-class citizen, in our world.\fP +It automatically creates and manages a virtualenv for your projects, as well as adds/removes packages from your \fBPipfile\fP as you install/uninstall packages. It also generates the ever\-important \fBPipfile.lock\fP, which is used to produce deterministic builds. .sp -It automatically creates and manages a virtualenv for your projects, as well as adds/removes packages from your \fBPipfile\fP as you install/uninstall packages. It also generates the ever-important \fBPipfile.lock\fP, which is used to produce deterministic builds. +Pipenv is primarily meant to provide users and developers of applications with an easy method to setup a working environment. For the distinction between libraries and applications and the usage of \fBsetup.py\fP vs \fBPipfile\fP to define dependencies, see pipfile\-vs\-setuppy\&. .sp The problems that Pipenv seeks to solve are multi\-faceted: .INDENT 0.0 .IP \(bu 2 You no longer need to use \fBpip\fP and \fBvirtualenv\fP separately. They work together. .IP \(bu 2 -Managing a \fBrequirements.txt\fP file \fI\%can be problematic\fP, so Pipenv uses the upcoming \fBPipfile\fP and \fBPipfile.lock\fP instead, which is superior for basic use cases. +Managing a \fBrequirements.txt\fP file \fI\%can be problematic\fP, so Pipenv uses \fBPipfile\fP and \fBPipfile.lock\fP to separate abstract dependency declarations from the last tested combination. .IP \(bu 2 Hashes are used everywhere, always. Security. Automatically expose security vulnerabilities. .IP \(bu 2 +Strongly encourage the use of the latest versions of dependencies to minimize security risks \fI\%arising from outdated components\fP\&. +.IP \(bu 2 Give you insight into your dependency graph (e.g. \fB$ pipenv graph\fP). .IP \(bu 2 Streamline development workflow by loading \fB\&.env\fP files. .UNINDENT -.SH INSTALL PIPENV TODAY! +.sp +You can quickly play with Pipenv right in your browser: +\fI\%Try in browser\fP.SH INSTALL PIPENV TODAY! +.sp +If you\(aqre on MacOS, you can install Pipenv easily with Homebrew: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ pip install pipenv -✨🍰✨ +$ brew install pipenv .ft P .fi .UNINDENT .UNINDENT .sp -If you have excellent taste, there\(aqs also a \fI\%fancy installation method\fP\&. +Or, if you\(aqre using Fedora 28: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ sudo dnf install pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Otherwise, refer to the installing\-pipenv chapter for instructions. +.sp +✨🍰✨ .SS Pipenv & Virtual Environments [image] .sp @@ -91,7 +110,7 @@ This guide is written for Python 3, however, these instructions should work fine on Python 2.7—if you are still using it, for some reason. .UNINDENT .UNINDENT -.SS ☤ Make sure you\(aqve got Python & pip +.SS ☤ Make sure you\(aqve got Python & pip .sp Before you go any further, make sure you have Python and that it\(aqs available from your command line. You can check this by simply running: @@ -150,8 +169,11 @@ pip 9.0.1 .UNINDENT .sp If you installed Python from source, with an installer from \fI\%python.org\fP, or -via \fI\%Homebrew\fP you should already have pip9. If you\(aqre on Linux and installed +via \fI\%Homebrew\fP you should already have pip. If you\(aqre on Linux and installed using your OS package manager, you may have to \fI\%install pip\fP separately. +.sp +If you plan to install Pipenv using Homebrew you can skip this step. The +Homebrew installer takes care of pip for you. .SS ☤ Installing Pipenv .sp Pipenv is a dependency manager for Python projects. If you\(aqre familiar @@ -159,8 +181,42 @@ with Node.js\(aq \fI\%npm\fP or Ruby\(aqs \fI\%bundler\fP, it is similar in spir tools. While pip can install Python packages, Pipenv is recommended as it\(aqs a higher\-level tool that simplifies dependency management for common use cases. +.SS ☤ Homebrew Installation of Pipenv .sp -Use \fBpip\fP to install Pipenv: +Homebrew is a popular open\-source package management system for macOS. +.sp +Installing pipenv via Homebrew will keep pipenv and all of its dependencies in +an isolated virtual environment so it doesn\(aqt interfere with the rest of your +Python installation. +.sp +Once you have installed \fI\%Homebrew\fP simply run: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ brew install pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To upgrade pipenv at any time: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ brew upgrade pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Pragmatic Installation of Pipenv +.sp +If you have a working installation of pip, and maintain certain "toolchain" type Python modules as global utilities in your user environment, pip \fI\%user installs\fP allow for installation into your home directory. Note that due to interaction between dependencies, you should limit tools installed in this way to basic building blocks for a Python workflow like virtualenv, pipenv, tox, and similar software. +.sp +To install: .INDENT 0.0 .INDENT 3.5 .sp @@ -187,13 +243,40 @@ absolute path to your home directory) so you\(aqll need to add \fI\%modifying ~/.profile\fP\&. .sp On Windows you can find the user base binary directory by running -\fBpy \-m site \-\-user\-site\fP and replacing \fBsite\-packages\fP with +\fBpython \-m site \-\-user\-site\fP and replacing \fBsite\-packages\fP with \fBScripts\fP\&. For example, this could return \fBC:\eUsers\eUsername\eAppData\eRoaming\ePython36\esite\-packages\fP so you would need to set your \fBPATH\fP to include \fBC:\eUsers\eUsername\eAppData\eRoaming\ePython36\eScripts\fP\&. You can set your user \fBPATH\fP permanently in the \fI\%Control Panel\fP\&. You may need to log out for the \fBPATH\fP changes to take effect. +.sp +For more information, see the \fI\%user installs documentation\fP\&. +.UNINDENT +.UNINDENT +.sp +To upgrade pipenv at any time: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pip install \-\-user \-\-upgrade pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Crude Installation of Pipenv +.sp +If you don\(aqt even have pip installed, you can use this crude installation method, which will bootstrap your whole system: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ curl https://raw.githubusercontent.com/kennethreitz/pipenv/master/get\-pipenv.py | python +.ft P +.fi .UNINDENT .UNINDENT .SS ☤ Installing packages for your project @@ -214,7 +297,7 @@ $ pipenv install requests .UNINDENT .sp Pipenv will install the excellent \fI\%Requests\fP library and create a \fBPipfile\fP -for you in your project\(aqs directory. The Pipfile is used to track which +for you in your project\(aqs directory. The \fBPipfile\fP is used to track which dependencies your project needs in case you need to re\-install them, such as when you share your project with others. You should get output similar to this (although the exact paths shown will vary): @@ -297,119 +380,742 @@ Your IP is 8.8.8.8 Using \fB$ pipenv run\fP ensures that your installed packages are available to your script. It\(aqs also possible to spawn a new shell that ensures all commands have access to your installed packages with \fB$ pipenv shell\fP\&. +.SS ☤ Virtualenv mapping caveat +.INDENT 0.0 +.IP \(bu 2 +Pipenv automatically maps projects to their specific virtualenvs. +.IP \(bu 2 +The virtualenv is stored globally with the name of the project’s root directory plus the hash of the full path to the project\(aqs root (e.g., \fBmy_project\-a3de50\fP). +.IP \(bu 2 +If you change your project\(aqs path, you break such a default mapping and pipenv will no longer be able to find and to use the project\(aqs virtualenv. +.IP \(bu 2 +You might want to set \fBexport PIPENV_VENV_IN_PROJECT=1\fP in your .bashrc/.zshrc (or any shell configuration file) for creating the virtualenv inside your project\(aqs directory, avoiding problems with subsequent path changes. +.UNINDENT .SS ☤ Next steps .sp Congratulations, you now know how to install and use Python packages! ✨ 🍰 ✨ -.SS ☤ Fancy Installation of Pipenv -.sp -To install pipenv in a fancy way, we recommend using \fI\%pipsi\fP\&. -.sp -Pipsi is a powerful tool which allows you to install Python scripts into isolated virtual environments. -.sp -To install pipsi, first run this: +.SS Release and Version History +.SS 2018.11.14 (2018\-11\-14) +.SS Features & Improvements .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ curl https://raw.githubusercontent.com/mitsuhiko/pipsi/master/get\-pipsi.py | python -.ft P -.fi +.IP \(bu 2 +Improved exceptions and error handling on failures. \fI\%#1977\fP +.IP \(bu 2 +Added persistent settings for all CLI flags via \fBPIPENV_{FLAG_NAME}\fP environment variables by enabling \fBauto_envvar_prefix=PIPENV\fP in click (implements PEEP\-0002). \fI\%#2200\fP +.IP \(bu 2 +Added improved messaging about available but skipped updates due to dependency conflicts when running \fBpipenv update \-\-outdated\fP\&. \fI\%#2411\fP +.IP \(bu 2 +Added environment variable \fIPIPENV_PYUP_API_KEY\fP to add ability +to override the bundled pyup.io API key. \fI\%#2825\fP +.IP \(bu 2 +Added additional output to \fBpipenv update \-\-outdated\fP to indicate that the operation succeded and all packages were already up to date. \fI\%#2828\fP +.IP \(bu 2 +Updated \fBcrayons\fP patch to enable colors on native powershell but swap native blue for magenta. \fI\%#3020\fP +.IP \(bu 2 +Added support for \fB\-\-bare\fP to \fBpipenv clean\fP, and fixed \fBpipenv sync \-\-bare\fP to actually reduce output. \fI\%#3041\fP +.IP \(bu 2 +Added windows\-compatible spinner via upgraded \fBvistir\fP dependency. \fI\%#3089\fP +.IP \(bu 2 +.INDENT 2.0 +.IP \(bu 2 +Added support for python installations managed by \fBasdf\fP\&. \fI\%#3096\fP .UNINDENT +.IP \(bu 2 +Improved runtime performance of no\-op commands such as \fBpipenv \-\-venv\fP by around 2/3. \fI\%#3158\fP +.IP \(bu 2 +Do not show error but success for running \fBpipenv uninstall \-\-all\fP in a fresh virtual environment. \fI\%#3170\fP +.IP \(bu 2 +Improved asynchronous installation and error handling via queued subprocess paralleization. \fI\%#3217\fP .UNINDENT -.sp -Follow the instructions, you\(aqll have to update your \fBPATH\fP\&. -.sp -Then, simply run: +.SS Bug Fixes .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pipsi install pipenv -.ft P -.fi +.IP \(bu 2 +Remote non\-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from \fBPyPI\fP\&. \fI\%#2394\fP +.IP \(bu 2 +Non\-ascii characters will now be handled correctly when parsed by pipenv\(aqs \fBToML\fP parsers. \fI\%#2737\fP +.IP \(bu 2 +Updated \fBpipenv uninstall\fP to respect the \fB\-\-skip\-lock\fP argument. \fI\%#2848\fP +.IP \(bu 2 +Fixed a bug which caused uninstallation to sometimes fail to successfullly remove packages from \fBPipfiles\fP with comments on preceding or following lines. \fI\%#2885\fP, +\fI\%#3099\fP +.IP \(bu 2 +Pipenv will no longer fail when encountering python versions on Windows that have been uninstalled. \fI\%#2983\fP +.IP \(bu 2 +Fixed unnecessary extras are added when translating markers \fI\%#3026\fP +.IP \(bu 2 +Fixed a virtualenv creation issue which could cause new virtualenvs to inadvertently attempt to read and write to global site packages. \fI\%#3047\fP +.IP \(bu 2 +Fixed an issue with virtualenv path derivation which could cause errors, particularly for users on WSL bash. \fI\%#3055\fP +.IP \(bu 2 +Fixed a bug which caused \fBUnexpected EOF\fP errors to be thrown when \fBpip\fP was waiting for input from users who had put login credentials in environment variables. \fI\%#3088\fP +.IP \(bu 2 +Fixed a bug in \fBrequirementslib\fP which prevented successful installation from mercurial repositories. \fI\%#3090\fP +.IP \(bu 2 +Fixed random resource warnings when using pyenv or any other subprocess calls. \fI\%#3094\fP +.IP \(bu 2 +.INDENT 2.0 +.IP \(bu 2 +Fixed a bug which sometimes prevented cloning and parsing \fBmercurial\fP requirements. \fI\%#3096\fP .UNINDENT +.IP \(bu 2 +Fixed an issue in \fBdelegator.py\fP related to subprocess calls when using \fBPopenSpawn\fP to stream output, which sometimes threw unexpected \fBEOF\fP errors. \fI\%#3102\fP, +\fI\%#3114\fP, +\fI\%#3117\fP +.IP \(bu 2 +Fix the path casing issue that makes \fIpipenv clean\fP fail on Windows \fI\%#3104\fP +.IP \(bu 2 +Pipenv will avoid leaving build artifacts in the current working directory. \fI\%#3106\fP +.IP \(bu 2 +Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures. \fI\%#3109\fP +.IP \(bu 2 +Fixed an issue which caused \fBpipenv clean\fP to sometimes clean packages from the base \fBsite\-packages\fP folder or fail entirely. \fI\%#3113\fP +.IP \(bu 2 +Updated \fBpythonfinder\fP to correct an issue with unnesting of nested paths when searching for python versions. \fI\%#3121\fP +.IP \(bu 2 +Added additional logic for ignoring and replacing non\-ascii characters when formatting console output on non\-UTF\-8 systems. \fI\%#3131\fP +.IP \(bu 2 +Fix virtual environment discovery when \fIPIPENV_VENV_IN_PROJECT\fP is set, but the in\-project \fI\&.venv\fP is a file. \fI\%#3134\fP +.IP \(bu 2 +Hashes for remote and local non\-PyPI artifacts will now be included in \fBPipfile.lock\fP during resolution. \fI\%#3145\fP +.IP \(bu 2 +Fix project path hashing logic in purpose to prevent collisions of virtual environments. \fI\%#3151\fP +.IP \(bu 2 +Fix package installation when the virtual environment path contains parentheses. \fI\%#3158\fP +.IP \(bu 2 +Azure Pipelines YAML files are updated to use the latest syntax and product name. \fI\%#3164\fP +.IP \(bu 2 +Fixed new spinner success message to write only one success message during resolution. \fI\%#3183\fP +.IP \(bu 2 +Pipenv will now correctly respect the \fB\-\-pre\fP option when used with \fBpipenv install\fP\&. \fI\%#3185\fP +.IP \(bu 2 +Fix a bug where exception is raised when run pipenv graph in a project without created virtualenv \fI\%#3201\fP +.IP \(bu 2 +When sources are missing names, names will now be derived from the supplied URL. \fI\%#3216\fP .UNINDENT -.sp -To upgrade pipenv at any time: +.SS Vendored Libraries .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pipsi upgrade pipenv -.ft P -.fi +.IP \(bu 2 +Updated \fBpythonfinder\fP to correct an issue with unnesting of nested paths when searching for python versions. \fI\%#3061\fP, +\fI\%#3121\fP +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBcertifi 2018.08.24 => 2018.10.15\fP +.IP \(bu 2 +\fBurllib3 1.23 => 1.24\fP +.IP \(bu 2 +\fBrequests 2.19.1 => 2.20.0\fP +.IP \(bu 2 +\fBshellingham \(ga\(ga1.2.6 => 1.2.7\fP +.IP \(bu 2 +\fBtomlkit 0.4.4. => 0.4.6\fP +.IP \(bu 2 +\fBvistir 0.1.6 => 0.1.8\fP +.IP \(bu 2 +\fBpythonfinder 0.1.2 => 0.1.3\fP +.IP \(bu 2 +\fBrequirementslib 1.1.9 => 1.1.10\fP +.IP \(bu 2 +\fBbackports.functools_lru_cache 1.5.0 (new)\fP +.IP \(bu 2 +\fBcursor 1.2.0 (new)\fP \fI\%#3089\fP .UNINDENT .UNINDENT -.sp -This will install \fBpipenv\fP in an isolated virtualenv, so it doesn\(aqt interfere with the rest of your Python installation! -.SS ☤ Pragmatic Installation of Pipenv -.sp -If you have a working installation of pip, and maintain certain "toolchain" type Python modules as global utilities in your user environment, pip \fI\%user installs\fP allow for installation into your home directory. Note that due to interaction between dependencies, you should limit tools installed in this way to basic building blocks for a Python workflow like virtualenv, pipenv, tox, and similar software. -.sp -To install: +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBrequests 2.19.1 => 2.20.1\fP +.IP \(bu 2 +\fBtomlkit 0.4.46 => 0.5.2\fP +.IP \(bu 2 +\fBvistir 0.1.6 => 0.2.4\fP +.IP \(bu 2 +\fBpythonfinder 1.1.2 => 1.1.8\fP +.IP \(bu 2 +\fBrequirementslib 1.1.10 => 1.3.0\fP \fI\%#3096\fP +.UNINDENT +.UNINDENT +.IP \(bu 2 +Switch to \fBtomlkit\fP for parsing and writing. Drop \fBprettytoml\fP and \fBcontoml\fP from vendors. \fI\%#3191\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to aid in resolution of local and remote archives. \fI\%#3196\fP +.UNINDENT +.SS Improved Documentation .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pip install \-\-user pipenv -.ft P -.fi +.IP \(bu 2 +Expanded development and testing documentation for contributors to get started. \fI\%#3074\fP .UNINDENT -.UNINDENT -.sp -For more information see the \fI\%user installs documentation\fP, but to add the installed cli tools from a pip user install to your path, add the output of: +.SS 2018.10.13 (2018\-10\-13) +.SS Bug Fixes .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ python \-c "import site; import os; print(os.path.join(site.USER_BASE, \(aqbin\(aq))" -.ft P -.fi +.IP \(bu 2 +Fixed a bug in \fBpipenv clean\fP which caused global packages to sometimes be inadvertently targeted for cleanup. \fI\%#2849\fP +.IP \(bu 2 +Fix broken backport imports for vendored vistir. \fI\%#2950\fP, +\fI\%#2955\fP, +\fI\%#2961\fP +.IP \(bu 2 +Fixed a bug with importing local vendored dependencies when running \fBpipenv graph\fP\&. \fI\%#2952\fP +.IP \(bu 2 +Fixed a bug which caused executable discovery to fail when running inside a virtualenv. \fI\%#2957\fP +.IP \(bu 2 +Fix parsing of outline tables. \fI\%#2971\fP +.IP \(bu 2 +Fixed a bug which caused \fBverify_ssl\fP to fail to drop through to \fBpip install\fP correctly as \fBtrusted\-host\fP\&. \fI\%#2979\fP +.IP \(bu 2 +Fixed a bug which caused canonicalized package names to fail to resolve against PyPI. \fI\%#2989\fP +.IP \(bu 2 +Enhanced CI detection to detect Azure Devops builds. \fI\%#2993\fP +.IP \(bu 2 +Fixed a bug which prevented installing pinned versions which used redirection symbols from the command line. \fI\%#2998\fP +.IP \(bu 2 +Fixed a bug which prevented installing the local directory in non\-editable mode. \fI\%#3005\fP .UNINDENT -.UNINDENT -.sp -To upgrade pipenv at any time: +.SS Vendored Libraries .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pip install \-\-user \-\-upgrade pipenv -.ft P -.fi +.IP \(bu 2 +Updated \fBrequirementslib\fP to version \fB1.1.9\fP\&. \fI\%#2989\fP +.IP \(bu 2 +Upgraded \fBpythonfinder => 1.1.1\fP and \fBvistir => 0.1.7\fP\&. \fI\%#3007\fP .UNINDENT -.UNINDENT -.SS ☤ Crude Installation of Pipenv -.sp -If you don\(aqt even have pip installed, you can use this crude installation method, which will bootstrap your whole system: +.SS 2018.10.9 (2018\-10\-09) +.SS Features & Improvements .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ curl https://raw.githubusercontent.com/kennethreitz/pipenv/master/get\-pipenv.py | python -.ft P -.fi +.IP \(bu 2 +Added environment variables \fIPIPENV_VERBOSE\fP and \fIPIPENV_QUIET\fP to control +output verbosity without needing to pass options. \fI\%#2527\fP +.IP \(bu 2 +Updated test\-pypi addon to better support json\-api access (forward compatibility). +Improved testing process for new contributors. \fI\%#2568\fP +.IP \(bu 2 +Greatly enhanced python discovery functionality: +.INDENT 2.0 +.IP \(bu 2 +Added pep514 (windows launcher/finder) support for python discovery. +.IP \(bu 2 +Introduced architecture discovery for python installations which support different architectures. \fI\%#2582\fP +.UNINDENT +.IP \(bu 2 +Added support for \fBpipenv shell\fP on msys and cygwin/mingw/git bash for Windows. \fI\%#2641\fP +.IP \(bu 2 +Enhanced resolution of editable and VCS dependencies. \fI\%#2643\fP +.IP \(bu 2 +Deduplicate and refactor CLI to use stateful arguments and object passing. See \fI\%this issue\fP for reference. \fI\%#2814\fP +.UNINDENT +.SS Behavior Changes +.INDENT 0.0 +.IP \(bu 2 +Virtual environment activation for \fBrun\fP is revised to improve interpolation +with other Python discovery tools. \fI\%#2503\fP +.IP \(bu 2 +Improve terminal coloring to display better in Powershell. \fI\%#2511\fP +.IP \(bu 2 +Invoke \fBvirtualenv\fP directly for virtual environment creation, instead of depending on \fBpew\fP\&. \fI\%#2518\fP +.IP \(bu 2 +\fBpipenv \-\-help\fP will now include short help descriptions. \fI\%#2542\fP +.IP \(bu 2 +Add \fBCOMSPEC\fP to fallback option (along with \fBSHELL\fP and \fBPYENV_SHELL\fP) +if shell detection fails, improving robustness on Windows. \fI\%#2651\fP +.IP \(bu 2 +Fallback to shell mode if \fIrun\fP fails with Windows error 193 to handle non\-executable commands. This should improve usability on Windows, where some users run non\-executable files without specifying a command, relying on Windows file association to choose the current command. \fI\%#2718\fP +.UNINDENT +.SS Bug Fixes +.INDENT 0.0 +.IP \(bu 2 +Fixed a bug which prevented installation of editable requirements using \fBssh://\fP style urls \fI\%#1393\fP +.IP \(bu 2 +VCS Refs for locked local editable dependencies will now update appropriately to the latest hash when running \fBpipenv update\fP\&. \fI\%#1690\fP +.IP \(bu 2 +\fB\&.tar.gz\fP and \fB\&.zip\fP artifacts will now have dependencies installed even when they are missing from the lockfile. \fI\%#2173\fP +.IP \(bu 2 +The command line parser will now handle multiple \fB\-e/\-\-editable\fP dependencies properly via click\(aqs option parser to help mitigate future parsing issues. \fI\%#2279\fP +.IP \(bu 2 +Fixed the ability of pipenv to parse \fBdependency_links\fP from \fBsetup.py\fP when \fBPIP_PROCESS_DEPENDENCY_LINKS\fP is enabled. \fI\%#2434\fP +.IP \(bu 2 +Fixed a bug which could cause \fB\-i/\-\-index\fP arguments to sometimes be incorrectly picked up in packages. This is now handled in the command line parser. \fI\%#2494\fP +.IP \(bu 2 +Fixed non\-deterministic resolution issues related to changes to the internal package finder in \fBpip 10\fP\&. \fI\%#2499\fP, +\fI\%#2529\fP, +\fI\%#2589\fP, +\fI\%#2666\fP, +\fI\%#2767\fP, +\fI\%#2785\fP, +\fI\%#2795\fP, +\fI\%#2801\fP, +\fI\%#2824\fP, +\fI\%#2862\fP, +\fI\%#2879\fP, +\fI\%#2894\fP, +\fI\%#2933\fP +.IP \(bu 2 +Fix subshell invocation on Windows for Python 2. \fI\%#2515\fP +.IP \(bu 2 +Fixed a bug which sometimes caused pipenv to throw a \fBTypeError\fP or to run into encoding issues when writing lockfiles on python 2. \fI\%#2561\fP +.IP \(bu 2 +Improve quoting logic for \fBpipenv run\fP so it works better with Windows +built\-in commands. \fI\%#2563\fP +.IP \(bu 2 +Fixed a bug related to parsing vcs requirements with both extras and subdirectory fragments. +Corrected an issue in the \fBrequirementslib\fP parser which led to some markers being discarded rather than evaluated. \fI\%#2564\fP +.IP \(bu 2 +Fixed multiple issues with finding the correct system python locations. \fI\%#2582\fP +.IP \(bu 2 +Catch JSON decoding error to prevent exception when the lock file is of +invalid format. \fI\%#2607\fP +.IP \(bu 2 +Fixed a rare bug which could sometimes cause errors when installing packages with custom sources. \fI\%#2610\fP +.IP \(bu 2 +Update requirementslib to fix a bug which could raise an \fBUnboundLocalError\fP when parsing malformed VCS URIs. \fI\%#2617\fP +.IP \(bu 2 +Fixed an issue which prevented passing multiple \fB\-\-ignore\fP parameters to \fBpipenv check\fP\&. \fI\%#2632\fP +.IP \(bu 2 +Fixed a bug which caused attempted hashing of \fBssh://\fP style URIs which could cause failures during installation of private ssh repositories. +\- Corrected path conversion issues which caused certain editable VCS paths to be converted to \fBssh://\fP URIs improperly. \fI\%#2639\fP +.IP \(bu 2 +Fixed a bug which caused paths to be formatted incorrectly when using \fBpipenv shell\fP in bash for windows. \fI\%#2641\fP +.IP \(bu 2 +Dependency links to private repositories defined via \fBssh://\fP schemes will now install correctly and skip hashing as long as \fBPIP_PROCESS_DEPENDENCY_LINKS=1\fP\&. \fI\%#2643\fP +.IP \(bu 2 +Fixed a bug which sometimes caused pipenv to parse the \fBtrusted_host\fP argument to pip incorrectly when parsing source URLs which specify \fBverify_ssl = false\fP\&. \fI\%#2656\fP +.IP \(bu 2 +Prevent crashing when a virtual environment in \fBWORKON_HOME\fP is faulty. \fI\%#2676\fP +.IP \(bu 2 +Fixed virtualenv creation failure when a .venv file is present in the project root. \fI\%#2680\fP +.IP \(bu 2 +Fixed a bug which could cause the \fB\-e/\-\-editable\fP argument on a dependency to be accidentally parsed as a dependency itself. \fI\%#2714\fP +.IP \(bu 2 +Correctly pass \fIverbose\fP and \fIdebug\fP flags to the resolver subprocess so it generates appropriate output. This also resolves a bug introduced by the fix to #2527. \fI\%#2732\fP +.IP \(bu 2 +All markers are now included in \fBpipenv lock \-\-requirements\fP output. \fI\%#2748\fP +.IP \(bu 2 +Fixed a bug in marker resolution which could cause duplicate and non\-deterministic markers. \fI\%#2760\fP +.IP \(bu 2 +Fixed a bug in the dependency resolver which caused regular issues when handling \fBsetup.py\fP based dependency resolution. \fI\%#2766\fP +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBpip\-tools\fP (updated and patched to latest w/ \fBpip 18.0\fP compatibilty) +.IP \(bu 2 +\fBpip 10.0.1 => 18.0\fP +.IP \(bu 2 +\fBclick 6.7 => 7.0\fP +.IP \(bu 2 +\fBtoml 0.9.4 => 0.10.0\fP +.IP \(bu 2 +\fBpyparsing 2.2.0 => 2.2.2\fP +.IP \(bu 2 +\fBdelegator 0.1.0 => 0.1.1\fP +.IP \(bu 2 +\fBattrs 18.1.0 => 18.2.0\fP +.IP \(bu 2 +\fBdistlib 0.2.7 => 0.2.8\fP +.IP \(bu 2 +\fBpackaging 17.1.0 => 18.0\fP +.IP \(bu 2 +\fBpassa 0.2.0 => 0.3.1\fP +.IP \(bu 2 +\fBpip_shims 0.1.2 => 0.3.1\fP +.IP \(bu 2 +\fBplette 0.1.1 => 0.2.2\fP +.IP \(bu 2 +\fBpythonfinder 1.0.2 => 1.1.0\fP +.IP \(bu 2 +\fBpytoml 0.1.18 => 0.1.19\fP +.IP \(bu 2 +\fBrequirementslib 1.1.16 => 1.1.17\fP +.IP \(bu 2 +\fBshellingham 1.2.4 => 1.2.6\fP +.IP \(bu 2 +\fBtomlkit 0.4.2 => 0.4.4\fP +.IP \(bu 2 +\fBvistir 0.1.4 => 0.1.6\fP \fI\%#2802\fP, .UNINDENT .UNINDENT .sp -Congratulations, you now have pip and Pipenv installed! +\fI\%#2867\fP, +\fI\%#2880\fP +.IP \(bu 2 +Fixed a bug where \fIpipenv\fP crashes when the \fIWORKON_HOME\fP directory does not exist. \fI\%#2877\fP +.IP \(bu 2 +Fixed pip is not loaded from pipenv\(aqs patched one but the system one \fI\%#2912\fP +.IP \(bu 2 +Fixed various bugs related to \fBpip 18.1\fP release which prevented locking, installation, and syncing, and dumping to a \fBrequirements.txt\fP file. \fI\%#2924\fP +.UNINDENT +.SS Vendored Libraries +.INDENT 0.0 +.IP \(bu 2 +Pew is no longer vendored. Entry point \fBpewtwo\fP, packages \fBpipenv.pew\fP and +\fBpipenv.patched.pew\fP are removed. \fI\%#2521\fP +.IP \(bu 2 +Update \fBpythonfinder\fP to major release \fB1.0.0\fP for integration. \fI\%#2582\fP +.IP \(bu 2 +Update requirementslib to fix a bug which could raise an \fBUnboundLocalError\fP when parsing malformed VCS URIs. \fI\%#2617\fP +.IP \(bu 2 +.INDENT 2.0 +.IP \(bu 2 +Vendored new libraries \fBvistir\fP and \fBpip\-shims\fP, \fBtomlkit\fP, \fBmodutil\fP, and \fBplette\fP\&. +.IP \(bu 2 +Update vendored libraries: +\- \fBscandir\fP to \fB1.9.0\fP +\- \fBclick\-completion\fP to \fB0.4.1\fP +\- \fBsemver\fP to \fB2.8.1\fP +\- \fBshellingham\fP to \fB1.2.4\fP +\- \fBpytoml\fP to \fB0.1.18\fP +\- \fBcertifi\fP to \fB2018.8.24\fP +\- \fBptyprocess\fP to \fB0.6.0\fP +\- \fBrequirementslib\fP to \fB1.1.5\fP +\- \fBpythonfinder\fP to \fB1.0.2\fP +\- \fBpipdeptree\fP to \fB0.13.0\fP +\- \fBpython\-dotenv\fP to \fB0.9.1\fP \fI\%#2639\fP +.UNINDENT +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBpip\-tools\fP (updated and patched to latest w/ \fBpip 18.0\fP compatibilty) +.IP \(bu 2 +\fBpip 10.0.1 => 18.0\fP +.IP \(bu 2 +\fBclick 6.7 => 7.0\fP +.IP \(bu 2 +\fBtoml 0.9.4 => 0.10.0\fP +.IP \(bu 2 +\fBpyparsing 2.2.0 => 2.2.2\fP +.IP \(bu 2 +\fBdelegator 0.1.0 => 0.1.1\fP +.IP \(bu 2 +\fBattrs 18.1.0 => 18.2.0\fP +.IP \(bu 2 +\fBdistlib 0.2.7 => 0.2.8\fP +.IP \(bu 2 +\fBpackaging 17.1.0 => 18.0\fP +.IP \(bu 2 +\fBpassa 0.2.0 => 0.3.1\fP +.IP \(bu 2 +\fBpip_shims 0.1.2 => 0.3.1\fP +.IP \(bu 2 +\fBplette 0.1.1 => 0.2.2\fP +.IP \(bu 2 +\fBpythonfinder 1.0.2 => 1.1.0\fP +.IP \(bu 2 +\fBpytoml 0.1.18 => 0.1.19\fP +.IP \(bu 2 +\fBrequirementslib 1.1.16 => 1.1.17\fP +.IP \(bu 2 +\fBshellingham 1.2.4 => 1.2.6\fP +.IP \(bu 2 +\fBtomlkit 0.4.2 => 0.4.4\fP +.IP \(bu 2 +\fBvistir 0.1.4 => 0.1.6\fP \fI\%#2902\fP, +.UNINDENT +.UNINDENT +.sp +\fI\%#2935\fP +.UNINDENT +.SS Improved Documentation +.INDENT 0.0 +.IP \(bu 2 +Simplified the test configuration process. \fI\%#2568\fP +.IP \(bu 2 +Updated documentation to use working fortune cookie addon. \fI\%#2644\fP +.IP \(bu 2 +Added additional information about troubleshooting \fBpipenv shell\fP by using the the \fB$PIPENV_SHELL\fP environment variable. \fI\%#2671\fP +.IP \(bu 2 +Added a link to \fBPEP\-440\fP version specifiers in the documentation for additional detail. \fI\%#2674\fP +.IP \(bu 2 +Added simple example to README.md for installing from git. \fI\%#2685\fP +.IP \(bu 2 +Stopped recommending \fI\-\-system\fP for Docker contexts. \fI\%#2762\fP +.IP \(bu 2 +Fixed the example url for doing "pipenv install \-e +some\-repo\-url#egg=something", it was missing the "egg=" in the fragment +identifier. \fI\%#2792\fP +.IP \(bu 2 +Fixed link to the "be cordial" essay in the contribution documentation. \fI\%#2793\fP +.IP \(bu 2 +Clarify \fIpipenv install\fP documentation \fI\%#2844\fP +.IP \(bu 2 +Replace reference to uservoice with PEEP\-000 \fI\%#2909\fP +.UNINDENT +.SS 2018.7.1 (2018\-07\-01) +.SS Features & Improvements +.INDENT 0.0 +.IP \(bu 2 +All calls to \fBpipenv shell\fP are now implemented from the ground up using \fI\%shellingham\fP, a custom library which was purpose built to handle edge cases and shell detection. \fI\%#2371\fP +.IP \(bu 2 +Added support for python 3.7 via a few small compatibility / bugfixes. \fI\%#2427\fP, +\fI\%#2434\fP, +\fI\%#2436\fP +.IP \(bu 2 +Added new flag \fBpipenv \-\-support\fP to replace the diagnostic command \fBpython \-m pipenv.help\fP\&. \fI\%#2477\fP, +\fI\%#2478\fP +.IP \(bu 2 +Improved import times and CLI runtimes with minor tweaks. \fI\%#2485\fP +.UNINDENT +.SS Bug Fixes +.INDENT 0.0 +.IP \(bu 2 +Fixed an ongoing bug which sometimes resolved incompatible versions into lockfiles. \fI\%#1901\fP +.IP \(bu 2 +Fixed a bug which caused errors when creating virtualenvs which contained leading dash characters. \fI\%#2415\fP +.IP \(bu 2 +Fixed a logic error which caused \fB\-\-deploy \-\-system\fP to overwrite editable vcs packages in the pipfile before installing, which caused any installation to fail by default. \fI\%#2417\fP +.IP \(bu 2 +Updated requirementslib to fix an issue with properly quoting markers in VCS requirements. \fI\%#2419\fP +.IP \(bu 2 +Installed new vendored jinja2 templates for \fBclick\-completion\fP which were causing template errors for users with completion enabled. \fI\%#2422\fP +.IP \(bu 2 +Added support for python 3.7 via a few small compatibility / bugfixes. \fI\%#2427\fP +.IP \(bu 2 +Fixed an issue reading package names from \fBsetup.py\fP files in projects which imported utilities such as \fBversioneer\fP\&. \fI\%#2433\fP +.IP \(bu 2 +Pipenv will now ensure that its internal package names registry files are written with unicode strings. \fI\%#2450\fP +.IP \(bu 2 +Fixed a bug causing requirements input as relative paths to be output as absolute paths or URIs. +Fixed a bug affecting normalization of \fBgit+git@host\fP uris. \fI\%#2453\fP +.IP \(bu 2 +Pipenv will now always use \fBpathlib2\fP for \fBPath\fP based filesystem interactions by default on \fBpython<3.5\fP\&. \fI\%#2454\fP +.IP \(bu 2 +Fixed a bug which prevented passing proxy PyPI indexes set with \fB\-\-pypi\-mirror\fP from being passed to pip during virtualenv creation, which could cause the creation to freeze in some cases. \fI\%#2462\fP +.IP \(bu 2 +Using the \fBpython \-m pipenv.help\fP command will now use proper encoding for the host filesystem to avoid encoding issues. \fI\%#2466\fP +.IP \(bu 2 +The new \fBjinja2\fP templates for \fBclick_completion\fP will now be included in pipenv source distributions. \fI\%#2479\fP +.IP \(bu 2 +Resolved a long\-standing issue with re\-using previously generated \fBInstallRequirement\fP objects for resolution which could cause \fBPKG\-INFO\fP file information to be deleted, raising a \fBTypeError\fP\&. \fI\%#2480\fP +.IP \(bu 2 +Resolved an issue parsing usernames from private PyPI URIs in \fBPipfiles\fP by updating \fBrequirementslib\fP\&. \fI\%#2484\fP +.UNINDENT +.SS Vendored Libraries +.INDENT 0.0 +.IP \(bu 2 +All calls to \fBpipenv shell\fP are now implemented from the ground up using \fI\%shellingham\fP, a custom library which was purpose built to handle edge cases and shell detection. \fI\%#2371\fP +.IP \(bu 2 +Updated requirementslib to fix an issue with properly quoting markers in VCS requirements. \fI\%#2419\fP +.IP \(bu 2 +Installed new vendored jinja2 templates for \fBclick\-completion\fP which were causing template errors for users with completion enabled. \fI\%#2422\fP +.IP \(bu 2 +Add patch to \fBprettytoml\fP to support Python 3.7. \fI\%#2426\fP +.IP \(bu 2 +Patched \fBprettytoml.AbstractTable._enumerate_items\fP to handle \fBStopIteration\fP errors in preparation of release of python 3.7. \fI\%#2427\fP +.IP \(bu 2 +Fixed an issue reading package names from \fBsetup.py\fP files in projects which imported utilities such as \fBversioneer\fP\&. \fI\%#2433\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to version \fB1.0.9\fP \fI\%#2453\fP +.IP \(bu 2 +Unraveled a lot of old, unnecessary patches to \fBpip\-tools\fP which were causing non\-deterministic resolution errors. \fI\%#2480\fP +.IP \(bu 2 +Resolved an issue parsing usernames from private PyPI URIs in \fBPipfiles\fP by updating \fBrequirementslib\fP\&. \fI\%#2484\fP +.UNINDENT +.SS Improved Documentation +.INDENT 0.0 +.IP \(bu 2 +Added instructions for installing using Fedora\(aqs official repositories. \fI\%#2404\fP +.UNINDENT +.SS 2018.6.25 (2018\-06\-25) +.SS Features & Improvements +.INDENT 0.0 +.IP \(bu 2 +Pipenv\-created virtualenvs will now be associated with a \fB\&.project\fP folder +(features can be implemented on top of this later or users may choose to use +\fBpipenv\-pipes\fP to take full advantage of this.) \fI\%#1861\fP +.IP \(bu 2 +Virtualenv names will now appear in prompts for most Windows users. \fI\%#2167\fP +.IP \(bu 2 +Added support for cmder shell paths with spaces. \fI\%#2168\fP +.IP \(bu 2 +Added nested JSON output to the \fBpipenv graph\fP command. \fI\%#2199\fP +.IP \(bu 2 +Dropped vendored pip 9 and vendored, patched, and migrated to pip 10. Updated +patched piptools version. \fI\%#2255\fP +.IP \(bu 2 +PyPI mirror URLs can now be set to override instances of PyPI urls by passing +the \fB\-\-pypi\-mirror\fP argument from the command line or setting the +\fBPIPENV_PYPI_MIRROR\fP environment variable. \fI\%#2281\fP +.IP \(bu 2 +Virtualenv activation lines will now avoid being written to some shell +history files. \fI\%#2287\fP +.IP \(bu 2 +Pipenv will now only search for \fBrequirements.txt\fP files when creating new +projects, and during that time only if the user doesn\(aqt specify packages to +pass in. \fI\%#2309\fP +.IP \(bu 2 +Added support for mounted drives via UNC paths. \fI\%#2331\fP +.IP \(bu 2 +Added support for Windows Subsystem for Linux bash shell detection. \fI\%#2363\fP +.IP \(bu 2 +Pipenv will now generate hashes much more quickly by resolving them in a +single pass during locking. \fI\%#2384\fP +.IP \(bu 2 +\fBpipenv run\fP will now avoid spawning additional \fBCOMSPEC\fP instances to +run commands in when possible. \fI\%#2385\fP +.IP \(bu 2 +Massive internal improvements to requirements parsing codebase, resolver, and +error messaging. \fI\%#2388\fP +.IP \(bu 2 +\fBpipenv check\fP now may take multiple of the additional argument +\fB\-\-ignore\fP which takes a parameter \fBcve_id\fP for the purpose of ignoring +specific CVEs. \fI\%#2408\fP +.UNINDENT +.SS Behavior Changes +.INDENT 0.0 +.IP \(bu 2 +Pipenv will now parse & capitalize \fBplatform_python_implementation\fP markers +.. warning:: This could cause an issue if you have an out of date \fBPipfile\fP +which lowercases the comparison value (e.g. \fBcpython\fP instead of +\fBCPython\fP). \fI\%#2123\fP +.IP \(bu 2 +Pipenv will now only search for \fBrequirements.txt\fP files when creating new +projects, and during that time only if the user doesn\(aqt specify packages to +pass in. \fI\%#2309\fP +.UNINDENT +.SS Bug Fixes +.INDENT 0.0 +.IP \(bu 2 +Massive internal improvements to requirements parsing codebase, resolver, and +error messaging. \fI\%#1962\fP, +\fI\%#2186\fP, +\fI\%#2263\fP, +\fI\%#2312\fP +.IP \(bu 2 +Pipenv will now parse & capitalize \fBplatform_python_implementation\fP +markers. \fI\%#2123\fP +.IP \(bu 2 +Fixed a bug with parsing and grouping old\-style \fBsetup.py\fP extras during +resolution \fI\%#2142\fP +.IP \(bu 2 +Fixed a bug causing pipenv graph to throw unhelpful exceptions when running +against empty or non\-existent environments. \fI\%#2161\fP +.IP \(bu 2 +Fixed a bug which caused \fB\-\-system\fP to incorrectly abort when users were in +a virtualenv. \fI\%#2181\fP +.IP \(bu 2 +Removed vendored \fBcacert.pem\fP which could cause issues for some users with +custom certificate settings. \fI\%#2193\fP +.IP \(bu 2 +Fixed a regression which led to direct invocations of \fBvirtualenv\fP, rather +than calling it by module. \fI\%#2198\fP +.IP \(bu 2 +Locking will now pin the correct VCS ref during \fBpipenv update\fP runs. +Running \fBpipenv update\fP with a new vcs ref specified in the \fBPipfile\fP +will now properly obtain, resolve, and install the specified dependency at +the specified ref. \fI\%#2209\fP +.IP \(bu 2 +\fBpipenv clean\fP will now correctly ignore comments from \fBpip freeze\fP when +cleaning the environment. \fI\%#2262\fP +.IP \(bu 2 +Resolution bugs causing packages for incompatible python versions to be +locked have been fixed. \fI\%#2267\fP +.IP \(bu 2 +Fixed a bug causing pipenv graph to fail to display sometimes. \fI\%#2268\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to fix a bug in pipfile parsing affecting +relative path conversions. \fI\%#2269\fP +.IP \(bu 2 +Windows executable discovery now leverages \fBos.pathext\fP\&. \fI\%#2298\fP +.IP \(bu 2 +Fixed a bug which caused \fB\-\-deploy \-\-system\fP to inadvertently create a +virtualenv before failing. \fI\%#2301\fP +.IP \(bu 2 +Fixed an issue which led to a failure to unquote special characters in file +and wheel paths. \fI\%#2302\fP +.IP \(bu 2 +VCS dependencies are now manually obtained only if they do not match the +requested ref. \fI\%#2304\fP +.IP \(bu 2 +Added error handling functionality to properly cope with single\-digit +\fBRequires\-Python\fP metatdata with no specifiers. \fI\%#2377\fP +.IP \(bu 2 +\fBpipenv update\fP will now always run the resolver and lock before ensuring +your dependencies are in sync with your lockfile. \fI\%#2379\fP +.IP \(bu 2 +Resolved a bug in our patched resolvers which could cause nondeterministic +resolution failures in certain conditions. Running \fBpipenv install\fP with no +arguments in a project with only a \fBPipfile\fP will now correctly lock first +for dependency resolution before installing. \fI\%#2384\fP +.IP \(bu 2 +Patched \fBpython\-dotenv\fP to ensure that environment variables always get +encoded to the filesystem encoding. \fI\%#2386\fP +.UNINDENT +.SS Improved Documentation +.INDENT 0.0 +.IP \(bu 2 +Update documentation wording to clarify Pipenv\(aqs overall role in the packaging ecosystem. \fI\%#2194\fP +.IP \(bu 2 +Added contribution documentation and guidelines. \fI\%#2205\fP +.IP \(bu 2 +Added instructions for supervisord compatibility. \fI\%#2215\fP +.IP \(bu 2 +Fixed broken links to development philosophy and contribution documentation. \fI\%#2248\fP +.UNINDENT +.SS Vendored Libraries +.INDENT 0.0 +.IP \(bu 2 +Removed vendored \fBcacert.pem\fP which could cause issues for some users with +custom certificate settings. \fI\%#2193\fP +.IP \(bu 2 +Dropped vendored pip 9 and vendored, patched, and migrated to pip 10. Updated +patched piptools version. \fI\%#2255\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to fix a bug in pipfile parsing affecting +relative path conversions. \fI\%#2269\fP +.IP \(bu 2 +Added custom shell detection library \fBshellingham\fP, a port of our changes +to \fBpew\fP\&. \fI\%#2363\fP +.IP \(bu 2 +Patched \fBpython\-dotenv\fP to ensure that environment variables always get +encoded to the filesystem encoding. \fI\%#2386\fP +.IP \(bu 2 +Updated vendored libraries. The following vendored libraries were updated: +.INDENT 2.0 +.IP \(bu 2 +distlib from version \fB0.2.6\fP to \fB0.2.7\fP\&. +.IP \(bu 2 +jinja2 from version \fB2.9.5\fP to \fB2.10\fP\&. +.IP \(bu 2 +pathlib2 from version \fB2.1.0\fP to \fB2.3.2\fP\&. +.IP \(bu 2 +parse from version \fB2.8.0\fP to \fB2.8.4\fP\&. +.IP \(bu 2 +pexpect from version \fB2.5.2\fP to \fB2.6.0\fP\&. +.IP \(bu 2 +requests from version \fB2.18.4\fP to \fB2.19.1\fP\&. +.IP \(bu 2 +idna from version \fB2.6\fP to \fB2.7\fP\&. +.IP \(bu 2 +certifi from version \fB2018.1.16\fP to \fB2018.4.16\fP\&. +.IP \(bu 2 +packaging from version \fB16.8\fP to \fB17.1\fP\&. +.IP \(bu 2 +six from version \fB1.10.0\fP to \fB1.11.0\fP\&. +.IP \(bu 2 +requirementslib from version \fB0.2.0\fP to \fB1.0.1\fP\&. +.UNINDENT +.sp +In addition, scandir was vendored and patched to avoid importing host system binaries when falling back to pathlib2. \fI\%#2368\fP +.UNINDENT .SH USER TESTIMONIALS .INDENT 0.0 .TP \fBJannis Leidel\fP, former pip maintainer— -\fIPipenv is the porcelain I always wanted to build for pip9. It fits my brain and mostly replaces virtualenvwrapper and manual pip calls for me. Use it.\fP +\fIPipenv is the porcelain I always wanted to build for pip. It fits my brain and mostly replaces virtualenvwrapper and manual pip calls for me. Use it.\fP +.TP +\fBDavid Gang\fP— +\fIThis package manager is really awesome. For the first time I know exactly what my dependencies are which I installed and what the transitive dependencies are. Combined with the fact that installs are deterministic, makes this package manager first class, like cargo\fP\&. .TP \fBJustin Myles Holmes\fP— \fIPipenv is finally an abstraction meant to engage the mind instead of merely the filesystem.\fP -.TP -\fBIsaac Sanders\fP— -\fIPipenv is literally the best thing about my day today. Thanks, Kenneth!\fP .UNINDENT .SH ☤ PIPENV FEATURES .INDENT 0.0 @@ -448,11 +1154,11 @@ Otherwise, whatever virtualenv defaults to will be the default. .SS Other Commands .INDENT 0.0 .IP \(bu 2 -\fBgraph\fP will show you a dependency graph, of your installed dependencies. +\fBgraph\fP will show you a dependency graph of your installed dependencies. .IP \(bu 2 -\fBshell\fP will spawn a shell with the virtualenv activated. +\fBshell\fP will spawn a shell with the virtualenv activated. This shell can be deactivated by using \fBexit\fP\&. .IP \(bu 2 -\fBrun\fP will run a given command from the virtualenv, with any arguments forwarded (e.g. \fB$ pipenv run python\fP). +\fBrun\fP will run a given command from the virtualenv, with any arguments forwarded (e.g. \fB$ pipenv run python\fP or \fB$ pipenv run pip freeze\fP). .IP \(bu 2 \fBcheck\fP checks for security vulnerabilities and asserts that PEP 508 requirements are being met by the current environment. .UNINDENT @@ -577,6 +1283,87 @@ pytest = "*" .fi .UNINDENT .UNINDENT +.SS ☤ General Recommendations & Version Control +.INDENT 0.0 +.IP \(bu 2 +Generally, keep both \fBPipfile\fP and \fBPipfile.lock\fP in version control. +.IP \(bu 2 +Do not keep \fBPipfile.lock\fP in version control if multiple versions of Python are being targeted. +.IP \(bu 2 +Specify your target Python version in your \fIPipfile\fP\(aqs \fB[requires]\fP section. Ideally, you should only have one target Python version, as this is a deployment tool. +.IP \(bu 2 +\fBpipenv install\fP is fully compatible with \fBpip install\fP syntax, for which the full documentation can be found \fI\%here\fP\&. +.UNINDENT +.SS ☤ Example Pipenv Workflow +.sp +Clone / create project repository: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ cd myproject +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Install from Pipfile, if there is one: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Or, add a package to your new project: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will create a \fBPipfile\fP if one doesn\(aqt exist. If one does exist, it will automatically be edited with the new package you provided. +.sp +Next, activate the Pipenv shell: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv shell +$ python \-\-version +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will spawn a new shell subprocess, which can be deactivated by using \fBexit\fP\&. +.SS ☤ Example Pipenv Upgrade Workflow +.INDENT 0.0 +.IP \(bu 2 +Find out what\(aqs changed upstream: \fB$ pipenv update \-\-outdated\fP\&. +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Upgrade packages, two options: +.INDENT 7.0 +.IP a. 3 +Want to upgrade everything? Just do \fB$ pipenv update\fP\&. +.IP b. 3 +Want to upgrade packages one\-at\-a\-time? \fB$ pipenv update \fP for each outdated package. +.UNINDENT +.UNINDENT +.UNINDENT .SS ☤ Importing from requirements.txt .sp If you only have a \fBrequirements.txt\fP file available when running \fBpipenv install\fP, @@ -584,23 +1371,71 @@ pipenv will automatically import the contents of this file and create a \fBPipfi .sp You can also specify \fB$ pipenv install \-r path/to/requirements.txt\fP to import a requirements file. .sp -Note, that when importing a requirements file, they often have version numbers pinned, which you likely won\(aqt want -in your \fBPipfile\fP, so you\(aqll have to manually update your \fBPipfile\fP afterwards to reflect this. +If your requirements file has version numbers pinned, you\(aqll likely want to edit the new \fBPipfile\fP +to remove those, and let \fBpipenv\fP keep track of pinning. If you want to keep the pinned versions +in your \fBPipfile.lock\fP for now, run \fBpipenv lock \-\-keep\-outdated\fP\&. Make sure to +\fI\%upgrade\fP soon! .SS ☤ Specifying Versions of a Package .sp -To tell pipenv to install a specific version of a library, the usage is simple: +You can specify versions of a package using the \fI\%Semantic Versioning scheme\fP +(i.e. \fBmajor.minor.micro\fP). +.sp +For example, to install requests you can use: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ pipenv install requests==2.13.0 +$ pipenv install requests~=1.2 # equivalent to requests~=1.2.0 .ft P .fi .UNINDENT .UNINDENT .sp +Pipenv will install version \fB1.2\fP and any minor update, but not \fB2.0\fP\&. +.sp This will update your \fBPipfile\fP to reflect this requirement, automatically. +.sp +In general, Pipenv uses the same specifier format as pip. However, note that according to \fI\%PEP 440\fP , you can\(aqt use versions containing a hyphen or a plus sign. +.sp +To make inclusive or exclusive version comparisons you can use: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install "requests>=1.4" # will install a version equal or larger than 1.4.0 +$ pipenv install "requests<=2.13" # will install a version equal or lower than 2.13.0 +$ pipenv install "requests>2.19" # will install 2.19.1 but not 2.19.0 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The use of \fB" "\fP around the package and version specification is highly recommended +to avoid issues with \fI\%Input and output redirection\fP +in Unix\-based operating systems. +.UNINDENT +.UNINDENT +.sp +The use of \fB~=\fP is preferred over the \fB==\fP identifier as the former prevents pipenv from updating the packages: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install "requests~=2.2" # locks the major version of the package (this is equivalent to using ==2.*) +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To avoid installing a specific version you can use the \fB!=\fP identifier. +.sp +For an in depth explanation of the valid identifiers and more complex use cases check \fI\%the relevant section of PEP\-440\fP\&. .SS ☤ Specifying Versions of Python .sp To create a new virtualenv, using a specific version of Python you have installed (and @@ -665,11 +1500,16 @@ python_version = "3.6" .UNINDENT .UNINDENT .sp -Note the inclusion of \fB[requires] python_version = "3.6"\fP\&. This specifies that your application requires this version +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The inclusion of \fB[requires] python_version = "3.6"\fP specifies that your application requires this version of Python, and will be used automatically when running \fBpipenv install\fP against this \fBPipfile\fP in the future (e.g. on other machines). If this is not true, feel free to simply remove this section. +.UNINDENT +.UNINDENT .sp -If you don\(aqt specify a Python version on the command-line, either the \fB[requires]\fP \fBpython_full_version\fP or \fBpython_version\fP will be selected +If you don\(aqt specify a Python version on the command–line, either the \fB[requires]\fP \fBpython_full_version\fP or \fBpython_version\fP will be selected automatically, falling back to whatever your system\(aqs default \fBpython\fP installation is, at time of execution. .SS ☤ Editable Dependencies (e.g. \fB\-e .\fP ) .sp @@ -680,17 +1520,25 @@ the current working directory when working on packages: .sp .nf .ft C -$ pipenv install \(aq\-e .\(aq \-\-dev +$ pipenv install \-\-dev \-e . $ cat Pipfile +\&... [dev\-packages] "e1839a8" = {path = ".", editable = true} +\&... .ft P .fi .UNINDENT .UNINDENT .sp -Note that all sub\-dependencies will get added to the \fBPipfile.lock\fP as well. +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +All sub\-dependencies will get added to the \fBPipfile.lock\fP as well. Sub\-dependencies are \fBnot\fP added to the +\fBPipfile.lock\fP if you leave the \fB\-e\fP option out. +.UNINDENT +.UNINDENT .SS ☤ Environment Management with Pipenv .sp The three primary commands you\(aqll use in managing your pipenv environment are @@ -745,7 +1593,7 @@ is unique. .UNINDENT .INDENT 0.0 .IP \(bu 2 -\fB\-\-dev\fP — Install both \fBdevelop\fP and \fBdefault\fP packages from \fBPipfile.lock\fP\&. +\fB\-\-dev\fP — Install both \fBdevelop\fP and \fBdefault\fP packages from \fBPipfile\fP\&. .IP \(bu 2 \fB\-\-system\fP — Use the system \fBpip\fP command rather than the one from your virtualenv. .IP \(bu 2 @@ -758,13 +1606,16 @@ is unique. .SS $ pipenv uninstall .sp \fB$ pipenv uninstall\fP supports all of the parameters in \fI\%pipenv install\fP, -as well as one additional, \fB\-\-all\fP\&. +as well as two additional options, \fB\-\-all\fP and \fB\-\-all\-dev\fP\&. .INDENT 0.0 .INDENT 3.5 .INDENT 0.0 .IP \(bu 2 \fB\-\-all\fP — This parameter will purge all files from the virtual environment, but leave the Pipfile untouched. +.IP \(bu 2 +\fB\-\-all\-dev\fP — This parameter will remove all of the development packages from +the virtual environment, and remove them from the Pipfile. .UNINDENT .UNINDENT .UNINDENT @@ -797,22 +1648,50 @@ You should do this for your shell too, in your \fB~/.profile\fP or \fB~/.bashrc\ The shell launched in interactive mode. This means that if your shell reads its configuration from a specific file for interactive mode (e.g. bash by default looks for a \fB~/.bashrc\fP configuration file for interactive mode), then you\(aqll need to modify (or create) this file. .UNINDENT .UNINDENT +.sp +If you experience issues with \fB$ pipenv shell\fP, just check the \fBPIPENV_SHELL\fP environment variable, which \fB$ pipenv shell\fP will use if available. For detail, see configuration\-with\-environment\-variables\&. .SS ☤ A Note about VCS Dependencies .sp -Pipenv will resolve the sub-dependencies of VCS dependencies, but only if they are editable, like so: +You can install packages with pipenv from git and other version control systems using URLs formatted according to the following rule: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[packages] -requests = {git = "https://github.com/requests/requests.git", editable=true} ++:////@#egg= .ft P .fi .UNINDENT .UNINDENT .sp -If editable is not true, sub-dependencies will not get resolved. +The only optional section is the \fB@\fP section. When using git over SSH, you may use the shorthand vcs and scheme alias \fBgit+git@:/@#\fP\&. Note that this is translated to \fBgit+ssh://git@\fP when parsed. +.sp +Note that it is \fBstrongly recommended\fP that you install any version\-controlled dependencies in editable mode, using \fBpipenv install \-e\fP, in order to ensure that dependency resolution can be performed with an up to date copy of the repository each time it is performed, and that it includes all known dependencies. +.sp +Below is an example usage which installs the git repository located at \fBhttps://github.com/requests/requests.git\fP from tag \fBv2.20.1\fP as package name \fBrequests\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-e git+https://github.com/requests/requests.git@v2.20.1#egg=requests +Creating a Pipfile for this project... +Installing \-e git+https://github.com/requests/requests.git@v2.20.1#egg=requests... +[...snipped...] +Adding \-e git+https://github.com/requests/requests.git@v2.20.1#egg=requests to Pipfile\(aqs [packages]... +[...] + +$ cat Pipfile +[packages] +requests = {git = "https://github.com/requests/requests.git", editable = true, ref = "v2.20.1"} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Valid values for \fB\fP include \fBgit\fP, \fBbzr\fP, \fBsvn\fP, and \fBhg\fP\&. Valid values for \fB\fP include \fBhttp\fP, \fBhttps\fP, \fBssh\fP, and \fBfile\fP\&. In specific cases you also have access to other schemes: \fBsvn\fP may be combined with \fBsvn\fP as a scheme, and \fBbzr\fP can be combined with \fBsftp\fP and \fBlp\fP\&. +.sp +You can read more about pip\(aqs implementation of VCS support \fI\%here\fP\&. For more information about other options available when specifying VCS dependencies, please check the \fI\%Pipfile spec\fP\&. .SS ☤ Pipfile.lock Security Features .sp \fBPipfile.lock\fP takes advantage of some great new security improvements in \fBpip\fP\&. @@ -824,10 +1703,28 @@ We highly recommend approaching deployments with promoting projects from a devel environment into production. You can use \fBpipenv lock\fP to compile your dependencies on your development environment and deploy the compiled \fBPipfile.lock\fP to all of your production environments for reproducible builds. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +If you\(aqd like a \fBrequirements.txt\fP output of the lockfile, run \fB$ pipenv lock \-r\fP\&. +This will include all hashes, however (which is great!). To get a \fBrequirements.txt\fP +without hashes, use \fB$ pipenv run pip freeze\fP\&. +.UNINDENT +.UNINDENT .SS Advanced Usage of Pipenv [image] .sp This document covers some of Pipenv\(aqs more glorious and advanced features. +.SS ☤ Caveats +.INDENT 0.0 +.IP \(bu 2 +Dependencies of wheels provided in a \fBPipfile\fP will not be captured by \fB$ pipenv lock\fP\&. +.IP \(bu 2 +There are some known issues with using private indexes, related to hashing. We\(aqre actively working to solve this problem. You may have great luck with this, however. +.IP \(bu 2 +Installation is intended to be as deterministic as possible —\ use the \fB\-\-sequential\fP flag to increase this, if experiencing issues. +.UNINDENT .SS ☤ Specifying Package Indexes .sp If you\(aqd like a specific package to be installed with a specific package index, you can do the following: @@ -858,6 +1755,50 @@ records = "*" .UNINDENT .sp Very fancy. +.SS ☤ Using a PyPI Mirror +.sp +If you\(aqd like to override the default PyPI index urls with the url for a PyPI mirror, you can use the following: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-\-pypi\-mirror + +$ pipenv update \-\-pypi\-mirror + +$ pipenv sync \-\-pypi\-mirror + +$ pipenv lock \-\-pypi\-mirror + +$ pipenv uninstall \-\-pypi\-mirror +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Alternatively, you can set the \fBPIPENV_PYPI_MIRROR\fP environment variable. +.SS ☤ Injecting credentials into Pipfiles via environment variables +.sp +Pipenv will expand environment variables (if defined) in your Pipfile. Quite +useful if you need to authenticate to a private PyPI: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[[source]] +url = "https://$USERNAME:${PASSWORD}@mypypi.example.com/simple" +verify_ssl = true +name = "pypi" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Luckily \- pipenv will hash your Pipfile \fIbefore\fP expanding environment +variables (and, helpfully, will substitute the environment variables again when +you install from the lock file \- so no need to commit any secrets! Woo!) .SS ☤ Specifying Basically Anything .sp If you\(aqd like to specify that a specific package only be installed on certain systems, @@ -876,7 +1817,7 @@ name = "pypi" [packages] requests = "*" -pywinusb = {version = "*", os_name = "== \(aqwindows\(aq"} +pywinusb = {version = "*", sys_platform = "== \(aqwin32\(aq"} .ft P .fi .UNINDENT @@ -902,9 +1843,45 @@ unittest2 = {version = ">=1.0,<3.0", markers="python_version < \(aq2.7.9\(aq or .UNINDENT .sp Magic. Pure, unadulterated magic. -.SS ☤ Deploying System Dependencies +.SS ☤ Using pipenv for Deployments .sp -You can tell Pipenv to install things into its parent system with the \fB\-\-system\fP flag: +You may want to use \fBpipenv\fP as part of a deployment process. +.sp +You can enforce that your \fBPipfile.lock\fP is up to date using the \fB\-\-deploy\fP flag: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-\-deploy +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will fail a build if the \fBPipfile.lock\fP is out–of–date, instead of generating a new one. +.sp +Or you can install packages exactly as specified in \fBPipfile.lock\fP using the \fBsync\fP command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv sync +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +\fBpipenv install \-\-ignore\-pipfile\fP is nearly equivalent to \fBpipenv sync\fP, but \fBpipenv sync\fP will \fInever\fP attempt to re\-lock your dependencies as it is considered an atomic operation. \fBpipenv install\fP by default does attempt to re\-lock unless using the \fB\-\-deploy\fP flag. +.UNINDENT +.UNINDENT +.SS Deploying System Dependencies +.sp +You can tell Pipenv to install a Pipfile\(aqs contents into its parent system with the \fB\-\-system\fP flag: .INDENT 0.0 .INDENT 3.5 .sp @@ -916,24 +1893,35 @@ $ pipenv install \-\-system .UNINDENT .UNINDENT .sp -This is useful for Docker containers, and deployment infrastructure (e.g. Heroku does this). +This is useful for managing the system Python, and deployment infrastructure (e.g. Heroku does this). +.SS ☤ Pipenv and Other Python Distributions .sp -Also useful for deployment is the \fB\-\-deploy\fP flag: +To use Pipenv with a third\-party Python distribution (e.g. Anaconda), you simply provide the path to the Python binary: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ pipenv install \-\-system \-\-deploy +$ pipenv install \-\-python=/path/to/python .ft P .fi .UNINDENT .UNINDENT .sp -This will fail a build if the \fBPipfile.lock\fP is out-of-date, instead of generating a new one. +Anaconda uses Conda to manage packages. To reuse Conda–installed Python packages, use the \fB\-\-site\-packages\fP flag: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv \-\-python=/path/to/python \-\-site\-packages +.ft P +.fi +.UNINDENT +.UNINDENT .SS ☤ Generating a \fBrequirements.txt\fP .sp -You can convert a \fBPipfile\fP and \fBPipenv.lock\fP into a \fBrequirements.txt\fP file very easily, and get all the benefits of hashes, extras, and other goodies we have included. +You can convert a \fBPipfile\fP and \fBPipfile.lock\fP into a \fBrequirements.txt\fP file very easily, and get all the benefits of extras and other goodies we have included. .sp Let\(aqs take this \fBPipfile\fP: .INDENT 0.0 @@ -959,11 +1947,42 @@ And generate a \fBrequirements.txt\fP out of it: .nf .ft C $ pipenv lock \-r -chardet==3.0.4 \-\-hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \-\-hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae -requests==2.18.4 \-\-hash=sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b \-\-hash=sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e -certifi==2017.7.27.1 \-\-hash=sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704 \-\-hash=sha256:40523d2efb60523e113b44602298f0960e900388cf3bb6043f645cf57ea9e3f5 -idna==2.6 \-\-hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4 \-\-hash=sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f -urllib3==1.22 \-\-hash=sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b \-\-hash=sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f +chardet==3.0.4 +requests==2.18.4 +certifi==2017.7.27.1 +idna==2.6 +urllib3==1.22 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +If you wish to generate a \fBrequirements.txt\fP with only the development requirements you can do that too! Let\(aqs take the following \fBPipfile\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true + +[dev\-packages] +pytest = {version="*"} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +And generate a \fBrequirements.txt\fP out of it: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv lock \-r \-\-dev +py==1.4.34 +pytest==3.2.3 .ft P .fi .UNINDENT @@ -1025,26 +2044,50 @@ hardened for production use and should be used only as a development aid. .UNINDENT .sp ✨🍰✨ -.SS ☤ Code Style Checking .sp -Pipenv has \fI\%Flake 8\fP built into it. You can check the style of your code like so, without installing anything: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +In order to enable this functionality while maintaining its permissive +copyright license, \fIpipenv\fP embeds an API client key for the backend +Safety API operated by pyup.io rather than including a full copy of the +CC\-BY\-NC\-SA licensed Safety\-DB database. This embedded client key is +shared across all \fIpipenv check\fP users, and hence will be subject to +API access throttling based on overall usage rather than individual +client usage. .sp -.nf -.ft C -$ cat t.py -import requests - -$ pipenv check \-\-style t.py -t.py:1:1: F401 \(aqrequests\(aq imported but unused -t.py:1:16: W292 no newline at end of file -.ft P -.fi +You can also use your own safety API key by setting the +environment variable \fBPIPENV_PYUP_API_KEY\fP\&. .UNINDENT .UNINDENT +.SS ☤ Community Integrations .sp -Super useful :) +There are a range of community\-maintained plugins and extensions available for a range of editors and IDEs, as well as +different products which integrate with Pipenv projects: +.INDENT 0.0 +.IP \(bu 2 +\fI\%Heroku\fP (Cloud Hosting) +.IP \(bu 2 +\fI\%Platform.sh\fP (Cloud Hosting) +.IP \(bu 2 +\fI\%PyUp\fP (Security Notification) +.IP \(bu 2 +\fI\%Emacs\fP (Editor Integration) +.IP \(bu 2 +\fI\%Fish Shell\fP (Automatic \fB$ pipenv shell\fP!) +.IP \(bu 2 +\fI\%VS Code\fP (Editor Integration) +.IP \(bu 2 +\fI\%PyCharm\fP (Editor Integration) +.UNINDENT +.sp +Works in progress: +.INDENT 0.0 +.IP \(bu 2 +\fI\%Sublime Text\fP (Editor Integration) +.IP \(bu 2 +Mysterious upcoming Google Cloud product (Cloud Hosting) +.UNINDENT .SS ☤ Open a Module in Your Editor .sp Pipenv allows you to open any Python module that is installed (including ones in your codebase), with the \fB$ pipenv open\fP command: @@ -1070,7 +2113,7 @@ This allows you to easily read the code you\(aqre consuming, instead of looking \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -The standard \fBEDITOR\fP environment variable is used for this. If you\(aqre using Sublime Text, for example, you\(aqll want to \fBexport EDITOR=subl\fP (once you\(aqve installed the command\-line utility). +The standard \fBEDITOR\fP environment variable is used for this. If you\(aqre using VS Code, for example, you\(aqll want to \fBexport EDITOR=code\fP (if you\(aqre on macOS you will want to \fI\%install the command\fP on to your \fBPATH\fP first). .UNINDENT .UNINDENT .SS ☤ Automatic Python Installation @@ -1157,37 +2200,297 @@ $ PIPENV_DOTENV_LOCATION=/path/to/.env pipenv shell .fi .UNINDENT .UNINDENT +.sp +To prevent pipenv from loading the \fB\&.env\fP file, set the \fBPIPENV_DONT_LOAD_ENV\fP environment variable: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ PIPENV_DONT_LOAD_ENV=1 pipenv shell +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Custom Script Shortcuts +.sp +Pipenv supports creating custom shortcuts in the (optional) \fB[scripts]\fP section of your Pipfile. +.sp +You can then run \fBpipenv run \fP in your terminal to run the command in the +context of your pipenv virtual environment even if you have not activated the pipenv shell first. +.sp +For example, in your Pipfile: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[scripts] +printspam = "python \-c \e"print(\(aqI am a silly example, no one would need to do this\(aq)\e"" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +And then in your terminal: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv run printspam +I am a silly example, no one would need to do this +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Commands that expect arguments will also work. +For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[scripts] +echospam = "echo I am really a very silly example" + +$ pipenv run echospam "indeed" +I am really a very silly example indeed +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Support for Environment Variables +.sp +Pipenv supports the usage of environment variables in values. For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[[source]] +url = "https://${PYPI_USERNAME}:${PYPI_PASSWORD}@my_private_repo.example.com/simple" +verify_ssl = true +name = "pypi" + +[dev\-packages] + +[packages] +requests = {version="*", index="home"} +maya = {version="*", index="pypi"} +records = "*" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Environment variables may be specified as \fB${MY_ENVAR}\fP or \fB$MY_ENVAR\fP\&. +On Windows, \fB%MY_ENVAR%\fP is supported in addition to \fB${MY_ENVAR}\fP or \fB$MY_ENVAR\fP\&. .SS ☤ Configuration With Environment Variables .sp -\fBpipenv\fP comes with a handful of options that can be enabled via shell environment +Pipenv comes with a handful of options that can be enabled via shell environment variables. To activate them, simply create the variable in your shell and pipenv will detect it. .INDENT 0.0 -.INDENT 3.5 +.TP +.B pipenv.environments.PIPENV_CACHE_DIR = \(aq/Users/fming/Library/Caches/pipenv\(aq +Location for Pipenv to store it\(aqs package cache. +.sp +Default is to use appdir\(aqs user cache directory. +.UNINDENT .INDENT 0.0 -.IP \(bu 2 -\fBPIPENV_DEFAULT_PYTHON_VERSION\fP — Use this version of Python when creating new virtual environments, by default (e.g. \fB3.6\fP). -.IP \(bu 2 -\fBPIPENV_SHELL_FANCY\fP — Always use fancy mode when invoking \fBpipenv shell\fP\&. -.IP \(bu 2 -\fBPIPENV_VENV_IN_PROJECT\fP — If set, use \fB\&.venv\fP in your project directory -instead of the global location. -.IP \(bu 2 -\fBPIPENV_COLORBLIND\fP — Disable terminal colors, for some reason. -.IP \(bu 2 -\fBPIPENV_NOSPIN\fP — Disable terminal spinner, for cleaner logs. Automatically set in CI environments. -.IP \(bu 2 -\fBPIPENV_MAX_DEPTH\fP — Set to an integer for the maximum number of directories to recursively -search for a Pipfile. -.IP \(bu 2 -\fBPIPENV_TIMEOUT\fP — Set to an integer for the max number of seconds Pipenv will -wait for virtualenv creation to complete. Defaults to 120 seconds. -.IP \(bu 2 -\fBPIPENV_IGNORE_VIRTUALENVS\fP — Set to disable automatically using an activated virtualenv over -the current project\(aqs own virtual environment. +.TP +.B pipenv.environments.PIPENV_COLORBLIND = False +If set, disable terminal colors. +.sp +Some people don\(aqt like colors in their terminals, for some reason. Default is +to show colors. .UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DEFAULT_PYTHON_VERSION = None +Use this Python version when creating new virtual environments by default. +.sp +This can be set to a version string, e.g. \fB3.6\fP, or a path. Default is to use +whatever Python Pipenv is installed under (i.e. \fBsys.executable\fP). Command +line flags (e.g. \fB\-\-python\fP, \fB\-\-three\fP, and \fB\-\-two\fP) are prioritized over +this configuration. .UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DONT_LOAD_ENV = False +If set, Pipenv does not load the \fB\&.env\fP file. +.sp +Default is to load \fB\&.env\fP for \fBrun\fP and \fBshell\fP commands. .UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DONT_USE_PYENV = False +If set, Pipenv does not attempt to install Python with pyenv. +.sp +Default is to install Python automatically via pyenv when needed, if possible. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DOTENV_LOCATION = None +If set, Pipenv loads the \fB\&.env\fP file at the specified location. +.sp +Default is to load \fB\&.env\fP from the project root, if found. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_EMULATOR = \(aq\(aq +If set, the terminal emulator\(aqs name for \fBpipenv shell\fP to use. +.sp +Default is to detect emulators automatically. This should be set if your +emulator, e.g. Cmder, cannot be detected correctly. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_HIDE_EMOJIS = False +Disable emojis in output. +.sp +Default is to show emojis. This is automatically set on Windows. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_IGNORE_VIRTUALENVS = False +If set, Pipenv will always assign a virtual environment for this project. +.sp +By default, Pipenv tries to detect whether it is run inside a virtual +environment, and reuses it if possible. This is usually the desired behavior, +and enables the user to use any user\-built environments with Pipenv. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_INSTALL_TIMEOUT = 900 +Max number of seconds to wait for package installation. +.sp +Defaults to 900 (15 minutes), a very long arbitrary time. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_DEPTH = 4 +Maximum number of directories to recursively search for a Pipfile. +.sp +Default is 3. See also \fBPIPENV_NO_INHERIT\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_RETRIES = 0 +Specify how many retries Pipenv should attempt for network requests. +.sp +Default is 0. Automatically set to 1 on CI environments for robust testing. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_ROUNDS = 16 +Tells Pipenv how many rounds of resolving to do for Pip\-Tools. +.sp +Default is 16, an arbitrary number that works most of the time. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_SUBPROCESS = 16 +How many subprocesses should Pipenv use when installing. +.sp +Default is 16, an arbitrary number that seems to work. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_NOSPIN = False +If set, disable terminal spinner. +.sp +This can make the logs cleaner. Automatically set on Windows, and in CI +environments. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_NO_INHERIT = False +Tell Pipenv not to inherit parent directories. +.sp +This is useful for deployment to avoid using the wrong current directory. +Overwrites \fBPIPENV_MAX_DEPTH\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_PIPFILE = None +If set, this specifies a custom Pipfile location. +.sp +When running pipenv from a location other than the same directory where the +Pipfile is located, instruct pipenv to find the Pipfile in the location +specified by this environment variable. +.sp +Default is to find Pipfile automatically in the current and parent directories. +See also \fBPIPENV_MAX_DEPTH\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_PYPI_MIRROR = None +If set, tells pipenv to override PyPI index urls with a mirror. +.sp +Default is to not mirror PyPI, i.e. use the real one, pypi.org. The +\fB\-\-pypi\-mirror\fP command line flag overwrites this. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SHELL = \(aq/bin/zsh\(aq +An absolute path to the preferred shell for \fBpipenv shell\fP\&. +.sp +Default is to detect automatically what shell is currently in use. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SHELL_FANCY = False +If set, always use fancy mode when invoking \fBpipenv shell\fP\&. +.sp +Default is to use the compatibility shell if possible. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SKIP_LOCK = False +If set, Pipenv won\(aqt lock dependencies automatically. +.sp +This might be desirable if a project has large number of dependencies, +because locking is an inherently slow operation. +.sp +Default is to lock dependencies and update \fBPipfile.lock\fP on each run. +.sp +NOTE: This only affects the \fBinstall\fP and \fBuninstall\fP commands. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SPINNER = \(aqdots\(aq +Sets the default spinner type. +.sp +Spinners are identitcal to the node.js spinners and can be found at +\fI\%https://github.com/sindresorhus/cli\-spinners\fP +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_TIMEOUT = 120 +Max number of seconds Pipenv will wait for virtualenv creation to complete. +.sp +Default is 120 seconds, an arbitrary number that seems to work. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_VENV_IN_PROJECT = False +If set, creates \fB\&.venv\fP in your project directory. +.sp +Default is to create new virtual environments in a global location. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_YES = False +If set, Pipenv automatically assumes "yes" at all prompts. +.sp +Default is to prompt the user for an answer if the current command line session +if interactive. +.UNINDENT +.sp +If you\(aqd like to set these environment variables on a per\-project basis, I recommend utilizing the fantastic \fI\%direnv\fP project, in order to do so. .sp Also note that \fI\%pip itself supports environment variables\fP, if you need additional customization. .sp @@ -1204,8 +2507,9 @@ $ PIP_INSTALL_OPTION="\-\- \-DCMAKE_BUILD_TYPE=Release" pipenv install \-e . .UNINDENT .SS ☤ Custom Virtual Environment Location .sp -Pipenv will automatically honor the \fBWORKON_HOME\fP environment -variable, if you have it set —\ so you can tell pipenv to store your virtual environments wherever you want, e.g.: +Pipenv automatically honors the \fBWORKON_HOME\fP environment variable, if you +have it set —\ so you can tell pipenv to store your virtual environments +wherever you want, e.g.: .INDENT 0.0 .INDENT 3.5 .sp @@ -1241,7 +2545,7 @@ python: \- "3.4" \- "3.5" \- "3.6" - \- "3.7dev" + \- "3.7\-dev" # command to install dependencies install: "make" @@ -1291,7 +2595,6 @@ commands= [testenv:flake8\-py3] basepython = python3.4 commands= - {[testenv]deps} pipenv install \-\-dev pipenv run flake8 \-\-version pipenv run flake8 setup.py docs project test @@ -1299,6 +2602,22 @@ commands= .fi .UNINDENT .UNINDENT +.sp +Pipenv will automatically use the virtualenv provided by \fBtox\fP\&. If \fBpipenv install \-\-dev\fP installs e.g. \fBpytest\fP, then installed command \fBpy.test\fP will be present in given virtualenv and can be called directly by \fBpy.test tests\fP instead of \fBpipenv run py.test tests\fP\&. +.sp +You might also want to add \fB\-\-ignore\-pipfile\fP to \fBpipenv install\fP, as to +not accidentally modify the lock\-file on each test run. This causes Pipenv +to ignore changes to the \fBPipfile\fP and (more importantly) prevents it from +adding the current environment to \fBPipfile.lock\fP\&. This might be important as +the current environment (i.e. the virtualenv provisioned by tox) will usually +contain the current project (which may or may not be desired) and additional +dependencies from \fBtox\fP\(aqs \fBdeps\fP directive. The initial provisioning may +alternatively be disabled by adding \fBskip_install = True\fP to tox.ini. +.sp +This method requires you to be explicit about updating the lock\-file, which is +probably a good idea in any case. +.sp +A 3rd party plugin, \fI\%tox\-pipenv\fP is also available to use Pipenv natively with tox. .SS ☤ Shell Completion .sp To enable completion in fish, add this to your config: @@ -1313,6 +2632,18 @@ eval (pipenv \-\-completion) .UNINDENT .UNINDENT .sp +Alternatively, with bash or zsh, add this to your config: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +eval "$(pipenv \-\-completion)" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp Magic shell completions are now enabled! .sp ✨🍰✨ @@ -1348,7 +2679,1213 @@ $ PIP_IGNORE_INSTALLED=1 pipenv install \-\-dev .fi .UNINDENT .UNINDENT +.SS ☤ Pipfile vs setup.py +.sp +There is a subtle but very important distinction to be made between \fBapplications\fP and \fBlibraries\fP\&. This is a very common source of confusion in the Python community. +.sp +Libraries provide reusable functionality to other libraries and applications (let\(aqs use the umbrella term \fBprojects\fP here). They are required to work alongside other libraries, all with their own set of subdependencies. They define \fBabstract dependencies\fP\&. To avoid version conflicts in subdependencies of different libraries within a project, libraries should never ever pin dependency versions. Although they may specify lower or (less frequently) upper bounds, if they rely on some specific feature/fix/bug. Library dependencies are specified via \fBinstall_requires\fP in \fBsetup.py\fP\&. +.sp +Libraries are ultimately meant to be used in some \fBapplication\fP\&. Applications are different in that they usually are not depended on by other projects. They are meant to be deployed into some specific environment and only then should the exact versions of all their dependencies and subdependencies be made concrete. To make this process easier is currently the main goal of Pipenv. +.sp +To summarize: +.INDENT 0.0 +.IP \(bu 2 +For libraries, define \fBabstract dependencies\fP via \fBinstall_requires\fP in \fBsetup.py\fP\&. The decision of which version exactly to be installed and where to obtain that dependency is not yours to make! +.IP \(bu 2 +For applications, define \fBdependencies and where to get them\fP in the \fIPipfile\fP and use this file to update the set of \fBconcrete dependencies\fP in \fBPipfile.lock\fP\&. This file defines a specific idempotent environment that is known to work for your project. The \fBPipfile.lock\fP is your source of truth. The \fBPipfile\fP is a convenience for you to create that lock\-file, in that it allows you to still remain somewhat vague about the exact version of a dependency to be used. Pipenv is there to help you define a working conflict\-free set of specific dependency\-versions, which would otherwise be a very tedious task. +.IP \(bu 2 +Of course, \fBPipfile\fP and Pipenv are still useful for library developers, as they can be used to define a development or test environment. +.IP \(bu 2 +And, of course, there are projects for which the distinction between library and application isn\(aqt that clear. In that case, use \fBinstall_requires\fP alongside Pipenv and \fBPipfile\fP\&. +.UNINDENT +.sp +You can also do this: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-e . +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will tell Pipenv to lock all your \fBsetup.py\fP–declared dependencies. +.SS ☤ Changing Pipenv\(aqs Cache Location +.sp +You can force Pipenv to use a different cache location by setting the environment variable \fBPIPENV_CACHE_DIR\fP to the location you wish. This is useful in the same situations that you would change \fBPIP_CACHE_DIR\fP to a different directory. +.SS ☤ Changing Default Python Versions +.sp +By default, Pipenv will initialize a project using whatever version of python the python3 is. Besides starting a project with the \fB\-\-three\fP or \fB\-\-two\fP flags, you can also use \fBPIPENV_DEFAULT_PYTHON_VERSION\fP to specify what version to use when starting a project when \fB\-\-three\fP or \fB\-\-two\fP aren\(aqt used. +.SS Frequently Encountered Pipenv Problems +.sp +Pipenv is constantly being improved by volunteers, but is still a very young +project with limited resources, and has some quirks that needs to be dealt +with. We need everyone’s help (including yours!). +.sp +Here are some common questions people have using Pipenv. Please take a look +below and see if they resolve your problem. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +\fBMake sure you’re running the newest Pipenv version first!\fP +.UNINDENT +.UNINDENT +.SS ☤ Your dependencies could not be resolved +.sp +Make sure your dependencies actually \fIdo\fP resolve. If you’re confident they +are, you may need to clear your resolver cache. Run the following command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv lock \-\-clear +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +and try again. +.sp +If this does not work, try manually deleting the whole cache directory. It is +usually one of the following locations: +.INDENT 0.0 +.IP \(bu 2 +\fB~/Library/Caches/pipenv\fP (macOS) +.IP \(bu 2 +\fB%LOCALAPPDATA%\epipenv\epipenv\eCache\fP (Windows) +.IP \(bu 2 +\fB~/.cache/pipenv\fP (other operating systems) +.UNINDENT +.sp +Pipenv does not install prereleases (i.e. a version with an alpha/beta/etc. +suffix, such as \fI1.0b1\fP) by default. You will need to pass the \fB\-\-pre\fP flag +in your command, or set +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[pipenv] +allow_prereleases = true +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +in your Pipfile. +.SS ☤ No module named +.sp +This is usually a result of mixing Pipenv with system packages. We \fIstrongly\fP +recommend installing Pipenv in an isolated environment. Uninstall all existing +Pipenv installations, and see installing\-pipenv to choose one of the +recommended way to install Pipenv instead. +.SS ☤ My pyenv\-installed Python is not found +.sp +Make sure you have \fBPYENV_ROOT\fP set correctly. Pipenv only supports CPython +distributions, with version name like \fB3.6.4\fP or similar. +.SS ☤ Pipenv does not respect pyenv’s global and local Python versions +.sp +Pipenv by default uses the Python it is installed against to create the +virtualenv. You can set the \fB\-\-python\fP option, or +\fB$PYENV_ROOT/shims/python\fP to let it consult pyenv when choosing the +interpreter. See specifying_versions for more information. +.sp +If you want Pipenv to automatically “do the right thing”, you can set the +environment variable \fBPIPENV_PYTHON\fP to \fB$PYENV_ROOT/shims/python\fP\&. This +will make Pipenv use pyenv’s active Python version to create virtual +environments by default. +.SS ☤ ValueError: unknown locale: UTF\-8 +.sp +macOS has a bug in its locale detection that prevents us from detecting your +shell encoding correctly. This can also be an issue on other systems if the +locale variables do not specify an encoding. +.sp +The workaround is to set the following two environment variables to a standard +localization format: +.INDENT 0.0 +.IP \(bu 2 +\fBLC_ALL\fP +.IP \(bu 2 +\fBLANG\fP +.UNINDENT +.sp +For Bash, for example, you can add the following to your \fB~/.bash_profile\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +export LC_ALL=\(aqen_US.UTF\-8\(aq +export LANG=\(aqen_US.UTF\-8\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +For Zsh, the file to edit is \fB~/.zshrc\fP\&. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +You can change both the \fBen_US\fP and \fBUTF\-8\fP part to the +language/locale and encoding you use. +.UNINDENT +.UNINDENT +.SS ☤ /bin/pip: No such file or directory +.sp +This may be related to your locale setting. See \fI\%☤ ValueError: unknown locale: UTF\-8\fP +for a possible solution. +.SS ☤ \fBshell\fP does not show the virtualenv’s name in prompt +.sp +This is intentional. You can do it yourself with either shell plugins, or +clever \fBPS1\fP configuration. If you really want it back, use +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv shell \-c +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +instead (not available on Windows). +.SS ☤ Pipenv does not respect dependencies in setup.py +.sp +No, it does not, intentionally. Pipfile and setup.py serve different purposes, +and should not consider each other by default. See pipfile\-vs\-setuppy +for more information. +.SS ☤ Using \fBpipenv run\fP in Supervisor program +.sp +When you configure a supervisor program\(aqs \fBcommand\fP with \fBpipenv run ...\fP, you +need to set locale enviroment variables properly to make it work. +.sp +Add this line under \fB[supervisord]\fP section in \fB/etc/supervisor/supervisord.conf\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[supervisord] +environment=LC_ALL=\(aqen_US.UTF\-8\(aq,LANG=\(aqen_US.UTF\-8\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ An exception is raised during \fBLocking dependencies…\fP +.sp +Run \fBpipenv lock \-\-clear\fP and try again. The lock sequence caches results +to speed up subsequent runs. The cache may contain faulty results if a bug +causes the format to corrupt, even after the bug is fixed. \fB\-\-clear\fP flushes +the cache, and therefore removes the bad results. +.SH CONTRIBUTION GUIDES +.SS Development Philosophy +.sp +Pipenv is an open but opinionated tool, created by an open but opinionated developer. +.SS Management Style +.sp +\fI\%Kenneth Reitz\fP is the BDFL. He has final say in any decision related to the Pipenv project. Kenneth is responsible for the direction and form of the library, as well as its presentation. In addition to making decisions based on technical merit, he is responsible for making decisions based on the development philosophy of Pipenv. +.sp +\fI\%Dan Ryan\fP, \fI\%Tzu\-ping Chung\fP, and \fI\%Nate Prewitt\fP are the core contributors. +They are responsible for triaging bug reports, reviewing pull requests and ensuring that Kenneth is kept up to speed with developments around the library. +The day\-to\-day managing of the project is done by the core contributors. They are responsible for making judgements about whether or not a feature request is +likely to be accepted by Kenneth. +.SS Values +.INDENT 0.0 +.IP \(bu 2 +Simplicity is always better than functionality. +.IP \(bu 2 +Listen to everyone, then disregard it. +.IP \(bu 2 +The API is all that matters. Everything else is secondary. +.IP \(bu 2 +Fit the 90% use\-case. Ignore the nay\-sayers. +.UNINDENT +.SS Contributing to Pipenv +.sp +If you\(aqre reading this, you\(aqre probably interested in contributing to Pipenv. +Thank you very much! Open source projects live\-and\-die based on the support +they receive from others, and the fact that you\(aqre even considering +contributing to the Pipenv project is \fIvery\fP generous of you. +.sp +This document lays out guidelines and advice for contributing to this project. +If you\(aqre thinking of contributing, please start by reading this document and +getting a feel for how contributing to this project works. If you have any +questions, feel free to reach out to either \fI\%Dan Ryan\fP, \fI\%Tzu\-ping Chung\fP, +or \fI\%Nate Prewitt\fP, the primary maintainers. +.sp +The guide is split into sections based on the type of contribution you\(aqre +thinking of making, with a section that covers general guidelines for all +contributors. +.SS Be Cordial +.INDENT 0.0 +.INDENT 3.5 +\fBBe cordial or be on your way\fP\&. \fI—Kenneth Reitz\fP +.UNINDENT +.UNINDENT +.sp +Pipenv has one very important rule governing all forms of contribution, +including reporting bugs or requesting features. This golden rule is +"\fI\%be cordial or be on your way\fP". +.sp +\fBAll contributions are welcome\fP, as long as +everyone involved is treated with respect. +.SS Get Early Feedback +.sp +If you are contributing, do not feel the need to sit on your contribution until +it is perfectly polished and complete. It helps everyone involved for you to +seek feedback as early as you possibly can. Submitting an early, unfinished +version of your contribution for feedback in no way prejudices your chances of +getting that contribution accepted, and can save you from putting a lot of work +into a contribution that is not suitable for the project. +.SS Contribution Suitability +.sp +Our project maintainers have the last word on whether or not a contribution is +suitable for Pipenv. All contributions will be considered carefully, but from +time to time, contributions will be rejected because they do not suit the +current goals or needs of the project. +.sp +If your contribution is rejected, don\(aqt despair! As long as you followed these +guidelines, you will have a much better chance of getting your next +contribution accepted. +.SS Code Contributions +.SS Steps for Submitting Code +.sp +When contributing code, you\(aqll want to follow this checklist: +.INDENT 0.0 +.IP 1. 3 +Fork the repository on GitHub. +.IP 2. 3 +\fI\%Run the tests\fP to confirm they all pass on your system. If they don\(aqt, you\(aqll +need to investigate why they fail. If you\(aqre unable to diagnose this +yourself, raise it as a bug report by following the guidelines in this +document: \fI\%Bug Reports\fP\&. +.IP 3. 3 +Write tests that demonstrate your bug or feature. Ensure that they fail. +.IP 4. 3 +Make your change. +.IP 5. 3 +Run the entire test suite again, confirming that all tests pass \fIincluding +the ones you just added\fP\&. +.IP 6. 3 +Send a GitHub Pull Request to the main repository\(aqs \fBmaster\fP branch. +GitHub Pull Requests are the expected method of code collaboration on this +project. +.UNINDENT +.sp +The following sub\-sections go into more detail on some of the points above. +.SS Code Review +.sp +Contributions will not be merged until they\(aqve been code reviewed. You should +implement any code review feedback unless you strongly object to it. In the +event that you object to the code review feedback, you should make your case +clearly and calmly. If, after doing so, the feedback is judged to still apply, +you must either apply the feedback or withdraw your contribution. +.SS Documentation Contributions +.sp +Documentation improvements are always welcome! The documentation files live in +the \fBdocs/\fP directory of the codebase. They\(aqre written in +\fI\%reStructuredText\fP, and use \fI\%Sphinx\fP to generate the full suite of +documentation. +.sp +When contributing documentation, please do your best to follow the style of the +documentation files. This means a soft\-limit of 79 characters wide in your text +files and a semi\-formal, yet friendly and approachable, prose style. +.sp +When presenting Python code, use single\-quoted strings (\fB\(aqhello\(aq\fP instead of +\fB"hello"\fP). +.SS Bug Reports +.sp +Bug reports are hugely important! Before you raise one, though, please check +through the \fI\%GitHub issues\fP, \fBboth open and closed\fP, to confirm that the bug +hasn\(aqt been reported before. Duplicate bug reports are a huge drain on the time +of other contributors, and should be avoided as much as possible. +.SS Run the tests +.sp +Three ways of running the tests are as follows: +.INDENT 0.0 +.IP 1. 3 +\fBmake test\fP (which uses \fBdocker\fP) +.IP 2. 3 +\fB\&./run\-tests.sh\fP or \fBrun\-tests.bat\fP +.IP 3. 3 +Using pipenv: +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv install \-\-dev +pipenv run pytest +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.sp +For the last two, it is important that your environment is setup correctly, and +this may take some work, for example, on a specific Mac installation, the following +steps may be needed: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# Make sure the tests can access github +if [ "$SSH_AGENT_PID" = "" ] +then + eval \(gassh\-agent\(ga + ssh\-add +fi + +# Use unix like utilities, installed with brew, +# e.g. brew install coreutils +for d in /usr/local/opt/*/libexec/gnubin /usr/local/opt/python/libexec/bin +do + [[ ":$PATH:" != *":$d:"* ]] && PATH="$d:${PATH}" +done + +export PATH + +# PIP_FIND_LINKS currently breaks test_uninstall.py +unset PIP_FIND_LINKS +.ft P +.fi +.UNINDENT +.UNINDENT .SH ☤ PIPENV USAGE +.SS pipenv +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv [OPTIONS] COMMAND [ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-where +Output project home information. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-venv +Output virtualenv information. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-py +Output Python interpreter information. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-envs +Output Environment Variable options. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-rm +Remove the virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-completion +Output completion (to be eval\(aqd). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-man +Display manpage. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-support +Output diagnostic information for use in GitHub issues. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-site\-packages +Enable site\-packages for the virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-version +Show the version and exit. +.UNINDENT +.SS check +.sp +Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv check [OPTIONS] [ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-unused +Given a code path, show potentially unused dependencies. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i, \-\-ignore +Ignore specified vulnerability during safety checks. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-system +System pip management. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B ARGS +Optional argument(s) +.UNINDENT +.SS clean +.sp +Uninstalls all packages not specified in Pipfile.lock. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv clean [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-dry\-run +Just output unneeded packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.SS graph +.sp +Displays currently\-installed dependency graph information. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv graph [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-json +Output JSON. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-json\-tree +Output JSON in nested tree. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-reverse +Reversed dependency graph. +.UNINDENT +.SS install +.sp +Installs provided packages and adds them to Pipfile, or (if no packages are given), installs all packages from Pipfile. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv install [OPTIONS] [PACKAGES]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-system +System pip management. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c, \-\-code +Import from codebase. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-deploy +Abort if the Pipfile.lock is out\-of\-date, or Python version is wrong. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-skip\-lock +Skip locking mechanisms and use the Pipfile instead during operation. +.UNINDENT +.INDENT 0.0 +.TP +.B \-e, \-\-editable +An editable python package URL or path, often to a VCS repo. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-ignore\-pipfile +Ignore Pipfile when installing, using the Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-selective\-upgrade +Update specified packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-r, \-\-requirements +Import a requirements.txt file. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-extra\-index\-url +URLs to the extra PyPI compatible indexes to query for package lookups. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i, \-\-index +Target PyPI\-compatible package index url. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-sequential +Install dependencies one\-at\-a\-time, instead of concurrently. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B PACKAGES +Optional argument(s) +.UNINDENT +Environment variables.INDENT 0.0 +.TP +.B PIPENV_SKIP_LOCK +Provide a default for \fI\%\-\-skip\-lock\fP +.UNINDENT +.INDENT 0.0 +.TP +.B PIP_EXTRA_INDEX_URL +Provide a default for \fI\%\-\-extra\-index\-url\fP +.UNINDENT +.INDENT 0.0 +.TP +.B PIP_INDEX_URL +Provide a default for \fI\%\-i\fP +.UNINDENT +.SS lock +.sp +Generates Pipfile.lock. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv lock [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-r, \-\-requirements +Generate output in requirements.txt format. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +.SS open +.sp +View a given module in your editor. +.sp +This uses the EDITOR environment variable. You can temporarily override it, +for example: +.INDENT 0.0 +.INDENT 3.5 +EDITOR=atom pipenv open requests +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv open [OPTIONS] MODULE +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B MODULE +Required argument +.UNINDENT +.SS run +.sp +Spawns a command installed into the virtualenv. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv run [OPTIONS] COMMAND [ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B COMMAND +Required argument +.UNINDENT +.INDENT 0.0 +.TP +.B ARGS +Optional argument(s) +.UNINDENT +.SS shell +.sp +Spawns a shell within the virtualenv. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv shell [OPTIONS] [SHELL_ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-fancy +Run in shell in fancy mode (for elegantly configured shells). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-anyway +Always spawn a subshell, even if one is already spawned. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B SHELL_ARGS +Optional argument(s) +.UNINDENT +.SS sync +.sp +Installs all packages specified in Pipfile.lock. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv sync [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-sequential +Install dependencies one\-at\-a\-time, instead of concurrently. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +.SS uninstall +.sp +Un\-installs a provided package and removes it from Pipfile. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv uninstall [OPTIONS] [PACKAGES]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-skip\-lock, \-\-lock +Lock afterwards. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-all\-dev +Un\-install all package from [dev\-packages]. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-all +Purge all package(s) from virtualenv. Does not edit Pipfile. +.UNINDENT +.INDENT 0.0 +.TP +.B \-e, \-\-editable +An editable python package URL or path, often to a VCS repo. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-skip\-lock +Skip locking mechanisms and use the Pipfile instead during operation. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B PACKAGES +Optional argument(s) +.UNINDENT +Environment variables.INDENT 0.0 +.TP +.B PIPENV_SKIP_LOCK +Provide a default for \fI\%\-\-skip\-lock\fP +.UNINDENT +.SS update +.sp +Runs lock, then sync. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv update [OPTIONS] [PACKAGES]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-outdated +List out\-of\-date dependencies. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-dry\-run +List out\-of\-date dependencies. +.UNINDENT +.INDENT 0.0 +.TP +.B \-e, \-\-editable +An editable python package URL or path, often to a VCS repo. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-ignore\-pipfile +Ignore Pipfile when installing, using the Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-selective\-upgrade +Update specified packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-r, \-\-requirements +Import a requirements.txt file. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-extra\-index\-url +URLs to the extra PyPI compatible indexes to query for package lookups. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i, \-\-index +Target PyPI\-compatible package index url. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-sequential +Install dependencies one\-at\-a\-time, instead of concurrently. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B PACKAGES +Optional argument(s) +.UNINDENT +Environment variables.INDENT 0.0 +.TP +.B PIP_EXTRA_INDEX_URL +Provide a default for \fI\%\-\-extra\-index\-url\fP +.UNINDENT +.INDENT 0.0 +.TP +.B PIP_INDEX_URL +Provide a default for \fI\%\-i\fP +.UNINDENT .INDENT 0.0 .IP \(bu 2 genindex diff --git a/tasks/release.py b/tasks/release.py index 4a242ba5..da13291e 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -62,6 +62,17 @@ def generate_markdown(ctx): ctx.run('pandoc CHANGELOG.rst -f rst -t markdown -o CHANGELOG.md') +@invoke.task +def generate_manual(ctx, commit=False): + log('Generating manual from reStructured source...') + ctx.run('make man -C docs') + ctx.run('cp docs/_build/man/pipenv.1 pipenv/') + if commit: + log('Commiting...') + ctx.run('git add pipenv/pipenv.1') + ctx.run('git commit -m "Update manual page."') + + @invoke.task def generate_changelog(ctx, commit=False, draft=False): log('Generating changelog...')