From cbe30d20c4e3c35a2dd5a8f39cca075f62ea0968 Mon Sep 17 00:00:00 2001 From: Mark Adamson Date: Wed, 27 Jun 2018 13:29:25 +0100 Subject: [PATCH 01/14] Fix typo in error message --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 7b30af90..a7ed3575 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1364,7 +1364,7 @@ def do_init( if not allow_global and not deploy and 'PIPENV_ACTIVE' not in os.environ: click.echo( "To activate this project's virtualenv, run {0}.\n" - "Alternativaly, run a command " + "Alternatively, run a command " "inside the virtualenv with {1}.".format( crayons.red('pipenv shell'), crayons.red('pipenv run'), From b6222e6f979a0fdbf3b82363c4e9e3aced5ead05 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 27 Jun 2018 16:12:25 -0400 Subject: [PATCH 02/14] Ensure we use unicode for proper names db Signed-off-by: Dan Ryan --- news/2450.bugfix | 1 + pipenv/project.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 news/2450.bugfix diff --git a/news/2450.bugfix b/news/2450.bugfix new file mode 100644 index 00000000..b9372832 --- /dev/null +++ b/news/2450.bugfix @@ -0,0 +1 @@ +Pipenv will now ensure that its internal package names registry files are written with unicode strings. diff --git a/pipenv/project.py b/pipenv/project.py index 0a9e04be..28c29eb4 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -363,7 +363,7 @@ class Project(object): def register_proper_name(self, name): """Registers a proper name to the database.""" with self.proper_names_db_path.open('a') as f: - f.write('{0}\n'.format(name)) + f.write(u'{0}\n'.format(name)) @property def pipfile_location(self): From 24743f77d394d83c9e80b181cda8f1d0bc04f436 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 27 Jun 2018 21:15:43 -0400 Subject: [PATCH 03/14] Switch to using pathlib2 for python<3.5 - Fixes #2438 Signed-off-by: Dan Ryan --- news/2454.bugfix | 1 + pipenv/_compat.py | 11 +++++++---- pipenv/project.py | 5 +---- pipenv/utils.py | 10 ++-------- 4 files changed, 11 insertions(+), 16 deletions(-) create mode 100644 news/2454.bugfix diff --git a/news/2454.bugfix b/news/2454.bugfix new file mode 100644 index 00000000..d60b047f --- /dev/null +++ b/news/2454.bugfix @@ -0,0 +1 @@ +Pipenv will now always use ``pathlib2`` for ``Path`` based filesystem interactions by default on ``python<3.5``. diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 68d8069d..07e75072 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -29,10 +29,13 @@ except ImportError: _types.add(type(arg)) return _types.pop() -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +if sys.version_info[:2] >= (3, 5): + try: + from pathlib import Path + except ImportError: + from .vendor.pathlib2 import Path +else: + from .vendor.pathlib2 import Path try: diff --git a/pipenv/project.py b/pipenv/project.py index d13f6491..9c8901e8 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -14,10 +14,7 @@ import six import toml import json as simplejson -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from ._compat import Path from .cmdparse import Script from .vendor.requirementslib import Requirement diff --git a/pipenv/utils.py b/pipenv/utils.py index a19f7ed0..12cd8283 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -36,13 +36,7 @@ try: from urllib.parse import urlparse except ImportError: from urlparse import urlparse -try: - from pathlib import Path -except ImportError: - try: - from .vendor.pathlib2 import Path - except ImportError: - pass + from distutils.spawn import find_executable from contextlib import contextmanager from .pep508checker import lookup @@ -231,7 +225,7 @@ def actually_resolve_deps( from pipenv.patched.piptools.scripts.compile import get_pip_command from pipenv.patched.piptools import logging as piptools_logging from pipenv.patched.piptools.exceptions import NoCandidateFound - from ._compat import TemporaryDirectory, NamedTemporaryFile + from ._compat import TemporaryDirectory, NamedTemporaryFile, Path class PipCommand(basecommand.Command): """Needed for pip-tools.""" From 39a363b780308fb83644fdecbb406fa34c8fc9a2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 27 Jun 2018 22:28:31 -0400 Subject: [PATCH 04/14] Update references to `Path` Signed-off-by: Dan Ryan --- tests/integration/conftest.py | 8 +------- tests/integration/test_install_basic.py | 6 +----- tests/integration/test_install_twists.py | 7 ++----- tests/integration/test_install_uri.py | 5 +---- tests/integration/test_pipenv.py | 8 ++------ tests/integration/test_windows.py | 6 +++--- 6 files changed, 10 insertions(+), 30 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6141eeac..d71f0f89 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,18 +4,12 @@ import warnings import pytest -from pipenv._compat import TemporaryDirectory +from pipenv._compat import TemporaryDirectory, Path from pipenv.vendor import delegator from pipenv.vendor import requests from pipenv.vendor import six from pipenv.vendor import toml -try: - from pathlib import Path -except ImportError: - from pipenv.vendor.pathlib2 import Path - - if six.PY2: class ResourceWarning(Warning): pass diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index 71e3ed5d..d4e967d4 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -2,13 +2,9 @@ import contextlib import os from pipenv.utils import temp_environ -from pipenv._compat import TemporaryDirectory +from pipenv._compat import TemporaryDirectory, Path from pipenv.vendor import delegator from pipenv.project import Project -try: - from pathlib import Path -except ImportError: - from pipenv.vendor.pathlib2 import Path import pytest diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 8750d463..95d67daf 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -1,10 +1,7 @@ import os import shutil from pipenv.project import Project -try: - import pathlib -except ImportError: - import pathlib2 as pathlib +from pipenv._compat import Path from pipenv.utils import mkdir_p, temp_environ @@ -223,7 +220,7 @@ def test_relative_paths(PipenvInstance, pypi, testsroot): dep = p.pipfile['packages'][key] assert 'path' in dep - assert pathlib.Path('.', artifact_dir, file_name) == pathlib.Path(dep['path']) + assert Path('.', artifact_dir, file_name) == Path(dep['path']) assert c.return_code == 0 diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 709ba8eb..df2d44d8 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -2,10 +2,7 @@ import pytest import os from flaky import flaky import delegator -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pipenv._compat import Path @pytest.mark.vcs diff --git a/tests/integration/test_pipenv.py b/tests/integration/test_pipenv.py index 7077d6e7..89ee793e 100644 --- a/tests/integration/test_pipenv.py +++ b/tests/integration/test_pipenv.py @@ -11,11 +11,7 @@ import pytest from pipenv.core import activate_virtualenv from pipenv.project import Project from pipenv.vendor import delegator - -try: - from pathlib import Path -except ImportError: - from pipenv.vendor.pathlib2 import Path +from pipenv._compat import Path @pytest.mark.code @@ -123,4 +119,4 @@ def test_directory_with_leading_dash(PipenvInstance): assert os.path.isdir(venv_path) # Manually clean up environment, since PipenvInstance assumes that # the virutalenv is in the project directory. - p.pipenv('--rm') \ No newline at end of file + p.pipenv('--rm') diff --git a/tests/integration/test_windows.py b/tests/integration/test_windows.py index 91b4f995..bd6eaf19 100644 --- a/tests/integration/test_windows.py +++ b/tests/integration/test_windows.py @@ -1,7 +1,7 @@ import os from pipenv.project import Project -from pipenv.vendor import pathlib2 as pathlib +from pipenv._compat import Path import pytest @@ -33,7 +33,7 @@ def test_case_changes_windows(PipenvInstance, pypi): @pytest.mark.files def test_local_path_windows(PipenvInstance, pypi): whl = ( - pathlib.Path(__file__).parent.parent + Path(__file__).parent.parent .joinpath('pypi', 'six', 'six-1.11.0-py2.py3-none-any.whl') ) try: @@ -48,7 +48,7 @@ def test_local_path_windows(PipenvInstance, pypi): @pytest.mark.files def test_local_path_windows_forward_slash(PipenvInstance, pypi): whl = ( - pathlib.Path(__file__).parent.parent + Path(__file__).parent.parent .joinpath('pypi', 'six', 'six-1.11.0-py2.py3-none-any.whl') ) try: From 7a014feb0d49d6ff8e27c84dc280053709a573c0 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 27 Jun 2018 22:36:26 -0400 Subject: [PATCH 05/14] Fix utils Signed-off-by: Dan Ryan --- pipenv/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 12cd8283..2068a227 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -14,7 +14,6 @@ import warnings from click import echo as click_echo from first import first - try: from weakref import finalize except ImportError: @@ -225,7 +224,7 @@ def actually_resolve_deps( from pipenv.patched.piptools.scripts.compile import get_pip_command from pipenv.patched.piptools import logging as piptools_logging from pipenv.patched.piptools.exceptions import NoCandidateFound - from ._compat import TemporaryDirectory, NamedTemporaryFile, Path + from ._compat import TemporaryDirectory, NamedTemporaryFile class PipCommand(basecommand.Command): """Needed for pip-tools.""" @@ -603,6 +602,7 @@ def is_installable_file(path): from .patched.notpip._internal.utils.misc import is_installable_dir from .patched.notpip._internal.utils.packaging import specifiers from .patched.notpip._internal.download import is_archive_file + from ._compat import Path if hasattr(path, 'keys') and any( key for key in path.keys() if key in ['file', 'path'] @@ -847,6 +847,7 @@ def find_windows_executable(bin_path, exe_name): def path_to_url(path): + from ._compat import Path return Path(normalize_drive(os.path.abspath(path))).as_uri() @@ -1152,7 +1153,7 @@ def get_vcs_deps( pypi_mirror=None, ): from .patched.notpip._internal.vcs import VcsSupport - from ._compat import TemporaryDirectory + from ._compat import TemporaryDirectory, Path section = "vcs_dev_packages" if dev else "vcs_packages" reqs = [] From ba9615cb7ec71a87b05b5b89a5de8403b0b55dc1 Mon Sep 17 00:00:00 2001 From: Max Krivich Date: Thu, 28 Jun 2018 18:11:15 +0300 Subject: [PATCH 06/14] Add `python_requires` in setup.py This argument prevents accident install `pipenv` on not supported python versions. Syntax for this string you can find in PEP440. Note: this feature works with pip 9.0+. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 9a31430d..0e0d9b7e 100644 --- a/setup.py +++ b/setup.py @@ -122,6 +122,7 @@ setup( "pipenv.patched.notpip._vendor.distlib._backport": ["sysconfig.cfg"], "pipenv.patched.notpip._vendor.distlib": ["t32.exe", "t64.exe", "w32.exe", "w64.exe"], }, + python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*', install_requires=required, extras_require={}, include_package_data=True, From f8396a1ecf8a69144749b5161a5e8acc23e95a30 Mon Sep 17 00:00:00 2001 From: Max Krivich Date: Thu, 28 Jun 2018 18:48:28 +0300 Subject: [PATCH 07/14] Add py3.3 to ignore list --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0e0d9b7e..d6290272 100644 --- a/setup.py +++ b/setup.py @@ -122,7 +122,7 @@ setup( "pipenv.patched.notpip._vendor.distlib._backport": ["sysconfig.cfg"], "pipenv.patched.notpip._vendor.distlib": ["t32.exe", "t64.exe", "w32.exe", "w64.exe"], }, - python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*', + python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*', install_requires=required, extras_require={}, include_package_data=True, From 4d43c39fb5aaac3b9822349f678484fbfb2ce6fe Mon Sep 17 00:00:00 2001 From: Max Krivich Date: Thu, 28 Jun 2018 19:00:09 +0300 Subject: [PATCH 08/14] Update classifiers in setup.py Remove python3.3 version from classifiers list --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index d6290272..2cbe33ff 100644 --- a/setup.py +++ b/setup.py @@ -132,7 +132,6 @@ setup( 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', From 0503c2046f857ec0756c6f5e215b2ec8d420f7c3 Mon Sep 17 00:00:00 2001 From: Jacob Henner Date: Thu, 28 Jun 2018 10:23:27 -0400 Subject: [PATCH 09/14] Use --pypi-mirror during virtualenv initialization Adds support for the --pypi-mirror parameter for all operations which may result in a virtualenv initialization. When a virtualenv is initialized, pip attempts to download several dependencies from PyPI. If PyPI is unavailable, virtualenv silently uses local packages instead, which is acceptable in most cases. However, in some environments connection attempts to PyPI will stall rather than fail, causing a pipenv timeout. By using the mirror specified by --pypi-mirror, we can ensure virtualenv will attempt to download dependencies from an accessible mirror instead of PyPI. - Fixes #2455 --- pipenv/cli.py | 58 +++++++++++++++++++++++++++++++++++++++++--------- pipenv/core.py | 35 ++++++++++++++++-------------- 2 files changed, 67 insertions(+), 26 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 5e7b470b..bb41c103 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -145,6 +145,13 @@ def validate_pypi_mirror(ctx, param, value): default=False, help="Enable site-packages for the virtualenv.", ) +@option( + '--pypi-mirror', + default=environments.PIPENV_PYPI_MIRROR, + nargs=1, + callback=validate_pypi_mirror, + help="Specify a PyPI mirror.", +) @version_option( prog_name=crayons.normal('pipenv', bold=True), version=__version__ ) @@ -163,6 +170,7 @@ def cli( envs=False, man=False, completion=False, + pypi_mirror=None, ): if completion: # Handle this ASAP to make shell startup fast. from . import shells @@ -272,7 +280,7 @@ def cli( # --two / --three was passed... if (python or three is not None) or site_packages: ensure_project( - three=three, python=python, warn=True, site_packages=site_packages + three=three, python=python, warn=True, site_packages=site_packages, pypi_mirror=pypi_mirror ) # Check this again before exiting for empty ``pipenv`` command. elif ctx.invoked_subcommand is None: @@ -571,7 +579,7 @@ def lock( from .core import ensure_project, do_init, do_lock # Ensure that virtualenv is available. - ensure_project(three=three, python=python) + ensure_project(three=three, python=python, pypi_mirror=pypi_mirror) if requirements: do_init(dev=dev, requirements=requirements, pypi_mirror=pypi_mirror) do_lock( @@ -608,9 +616,16 @@ def lock( default=False, help="Always spawn a subshell, even if one is already spawned.", ) +@option( + '--pypi-mirror', + default=environments.PIPENV_PYPI_MIRROR, + nargs=1, + callback=validate_pypi_mirror, + help="Specify a PyPI mirror.", +) @argument('shell_args', nargs=-1) def shell( - three=None, python=False, fancy=False, shell_args=None, anyway=False + three=None, python=False, fancy=False, shell_args=None, anyway=False, pypi_mirror=None ): from .core import load_dot_env, do_shell # Prevent user from activating nested environments. @@ -635,7 +650,7 @@ def shell( if os.name == 'nt': fancy = True do_shell( - three=three, python=python, fancy=fancy, shell_args=shell_args + three=three, python=python, fancy=fancy, shell_args=shell_args, pypi_mirror=pypi_mirror ) @@ -663,9 +678,16 @@ def shell( callback=validate_python_path, help="Specify which version of Python virtualenv should use.", ) -def run(command, args, three=None, python=False): +@option( + '--pypi-mirror', + default=environments.PIPENV_PYPI_MIRROR, + nargs=1, + callback=validate_pypi_mirror, + help="Specify a PyPI mirror.", +) +def run(command, args, three=None, python=False, pypi_mirror=None): from .core import do_run - do_run(command=command, args=args, three=three, python=python) + do_run(command=command, args=args, three=three, python=python, pypi_mirror=pypi_mirror) @command( @@ -700,6 +722,13 @@ def run(command, args, three=None, python=False): multiple=True, help="Ignore specified vulnerability during safety checks." ) +@option( + '--pypi-mirror', + default=environments.PIPENV_PYPI_MIRROR, + nargs=1, + callback=validate_pypi_mirror, + help="Specify a PyPI mirror.", +) @argument('args', nargs=-1) def check( three=None, @@ -709,6 +738,7 @@ def check( style=False, ignore=None, args=None, + pypi_mirror=None, ): from .core import do_check do_check( @@ -717,7 +747,8 @@ def check( system=system, unused=unused, ignore=ignore, - args=args + args=args, + pypi_mirror=pypi_mirror ) @@ -819,7 +850,7 @@ def update( project, ) - ensure_project(three=three, python=python, warn=True) + ensure_project(three=three, python=python, warn=True, pypi_mirror=pypi_mirror) if not outdated: outdated = bool(dry_run) if outdated: @@ -894,12 +925,19 @@ def graph(bare=False, json=False, json_tree=False, reverse=False): callback=validate_python_path, help="Specify which version of Python virtualenv should use.", ) +@option( + '--pypi-mirror', + default=environments.PIPENV_PYPI_MIRROR, + nargs=1, + callback=validate_pypi_mirror, + help="Specify a PyPI mirror.", +) @argument('module', nargs=1) -def run_open(module, three=None, python=None): +def run_open(module, three=None, python=None, pypi_mirror=None): from .core import which, ensure_project # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False) + ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) c = delegator.run( '{0} -c "import {1}; print({1}.__file__);"'.format( which('python'), module diff --git a/pipenv/core.py b/pipenv/core.py index 7b30af90..7dba3c9d 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -548,7 +548,7 @@ def ensure_python(three=None, python=None): return path_to_python -def ensure_virtualenv(three=None, python=None, site_packages=False): +def ensure_virtualenv(three=None, python=None, site_packages=False, pypi_mirror=None): """Creates a virtualenv, if one doesn't exist.""" def abort(): @@ -571,7 +571,7 @@ def ensure_virtualenv(three=None, python=None, site_packages=False): ) ) sys.exit(1) - do_create_virtualenv(python=python, site_packages=site_packages) + do_create_virtualenv(python=python, site_packages=site_packages, pypi_mirror=pypi_mirror) except KeyboardInterrupt: # If interrupted, cleanup the virtualenv. cleanup_virtualenv(bare=False) @@ -599,7 +599,7 @@ def ensure_virtualenv(three=None, python=None, site_packages=False): cleanup_virtualenv(bare=True) # Call this function again. ensure_virtualenv( - three=three, python=python, site_packages=site_packages + three=three, python=python, site_packages=site_packages, pypi_mirror=pypi_mirror ) @@ -612,6 +612,7 @@ def ensure_project( site_packages=False, deploy=False, skip_requirements=False, + pypi_mirror=None, ): """Ensures both Pipfile and virtualenv exist for the project.""" # Automatically use an activated virtualenv. @@ -622,7 +623,7 @@ def ensure_project( # Skip virtualenv creation when --system was used. if not system: ensure_virtualenv( - three=three, python=python, site_packages=site_packages + three=three, python=python, site_packages=site_packages, pypi_mirror=pypi_mirror ) if warn: # Warn users if they are using the wrong version of Python. @@ -885,7 +886,7 @@ def convert_three_to_python(three, python): return python -def do_create_virtualenv(python=None, site_packages=False): +def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): """Creates a virtualenv.""" click.echo( crayons.normal(u'Creating a virtualenv for this project...', bold=True), @@ -933,7 +934,8 @@ def do_create_virtualenv(python=None, site_packages=False): # Actually create the virtualenv. with spinner(): try: - c = delegator.run(cmd, block=False, timeout=PIPENV_TIMEOUT) + pip_config = {'PIP_INDEX_URL': fs_str(pypi_mirror)} if pypi_mirror else {} + c = delegator.run(cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config) except OSError: click.echo( '{0}: it looks like {1} is not in your {2}. ' @@ -1264,7 +1266,7 @@ def do_init( if not system: if not project.virtualenv_exists: try: - do_create_virtualenv() + do_create_virtualenv(pypi_mirror=pypi_mirror) except KeyboardInterrupt: cleanup_virtualenv(bare=False) sys.exit(1) @@ -1789,6 +1791,7 @@ def do_install( warn=True, deploy=deploy, skip_requirements=skip_requirements, + pypi_mirror=pypi_mirror, ) # Load the --pre settings from the Pipfile. if not pre: @@ -2097,7 +2100,7 @@ def do_uninstall( if PIPENV_USE_SYSTEM: system = True # Ensure that virtualenv is available. - ensure_project(three=three, python=python) + ensure_project(three=three, python=python, pypi_mirror=pypi_mirror) package_names = (package_name,) + more_packages pipfile_remove = True # Un-install all dependencies, if --all was provided. @@ -2166,11 +2169,11 @@ def do_uninstall( do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) -def do_shell(three=None, python=False, fancy=False, shell_args=None): +def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror=None): from .patched.pew import pew # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False) + ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) # Set an environment variable, so we know we're in the environment. os.environ['PIPENV_ACTIVE'] = '1' compat = (not fancy) @@ -2320,13 +2323,13 @@ def do_run_posix(script, command): os.execl(command_path, command_path, *script.args) -def do_run(command, args, three=None, python=False): +def do_run(command, args, three=None, python=False, pypi_mirror=None): """Attempt to run command either pulling from project or interpreting as executable. Args are appended to the command in [scripts] section of project if found. """ # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False) + ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) load_dot_env() # Activate virtualenv under the current interpreter's environment inline_activate_virtualenv() @@ -2340,10 +2343,10 @@ def do_run(command, args, three=None, python=False): do_run_posix(script, command=command) -def do_check(three=None, python=False, system=False, unused=False, ignore=None, args=None): +def do_check(three=None, python=False, system=False, unused=False, ignore=None, args=None, pypi_mirror=None): if not system: # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False, warn=False) + ensure_project(three=three, python=python, validate=False, warn=False, pypi_mirror=pypi_mirror) if not args: args = [] if unused: @@ -2586,7 +2589,7 @@ def do_sync( sys.exit(1) # Ensure that virtualenv is available if not system. - ensure_project(three=three, python=python, validate=False, deploy=deploy) + ensure_project(three=three, python=python, validate=False, deploy=deploy, pypi_mirror=pypi_mirror) # Install everything. requirements_dir = TemporaryDirectory( @@ -2610,7 +2613,7 @@ def do_clean( ctx, three=None, python=None, dry_run=False, bare=False, verbose=False, pypi_mirror=None ): # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False) + ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) ensure_lockfile(pypi_mirror=pypi_mirror) installed_package_names = [] From ed3de3f2dece0459b68bc687dcb6293c78d16635 Mon Sep 17 00:00:00 2001 From: Max Krivich Date: Thu, 28 Jun 2018 21:47:38 +0300 Subject: [PATCH 10/14] Fix `UnicodeEncodeError` in help.py --- pipenv/help.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pipenv/help.py b/pipenv/help.py index e08c6173..0d06680d 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -9,6 +9,13 @@ from .core import project, system_which, find_python_in_path, python_version from .pep508checker import lookup +def print_utf(line): + try: + print(line) + except UnicodeEncodeError: + print(line.encode('utf-8')) + + def main(): print('
$ python -m pipenv.help output') print('') @@ -45,13 +52,13 @@ def main(): for key in os.environ: print(' - `{0}`'.format(key)) print('') - print(u'Pipenv–specific environment variables:') + print_utf(u'Pipenv–specific environment variables:') print('') for key in os.environ: if key.startswith('PIPENV'): print(' - `{0}`: `{1}`'.format(key, os.environ[key])) print('') - print(u'Debug–specific environment variables:') + print_utf(u'Debug–specific environment variables:') print('') for key in ('PATH', 'SHELL', 'EDITOR', 'LANG', 'PWD', 'VIRTUAL_ENV'): if key in os.environ: @@ -61,7 +68,7 @@ def main(): print('---------------------------') print('') if project.pipfile_exists: - print( + print_utf( u'Contents of `Pipfile` ({0!r}):'.format(project.pipfile_location) ) print('') @@ -72,7 +79,7 @@ def main(): print('') if project.lockfile_exists: print('') - print( + print_utf( u'Contents of `Pipfile.lock` ({0!r}):'.format( project.lockfile_location ) @@ -87,3 +94,4 @@ def main(): if __name__ == '__main__': main() + From 19640af9b7b6f2cc8630e09760c7951f797520f4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 27 Jun 2018 21:02:18 -0400 Subject: [PATCH 11/14] Update vendored dependencies - Updated requirementslib - Fix local relative path bug whici caused paths to be output as absolute (test added to requirementslib) - Fix SSH normalization bug (test added to requirementslib) - Fixes #2440, #2441 Signed-off-by: Dan Ryan --- news/2453.bugfix | 2 + news/2453.vendor | 1 + pipenv/patched/piptools/utils.py | 2 +- pipenv/vendor/requirementslib/__init__.py | 2 +- .../requirementslib/models/requirements.py | 22 +++++++---- pipenv/vendor/requirementslib/models/utils.py | 2 +- pipenv/vendor/shellingham/__init__.py | 2 +- pipenv/vendor/shellingham/posix/__init__.py | 39 +++++++++++++++++++ pipenv/vendor/shellingham/posix/_default.py | 27 +++++++++++++ pipenv/vendor/shellingham/posix/linux.py | 35 +++++++++++++++++ pipenv/vendor/vendor.txt | 2 +- tasks/vendoring/__init__.py | 2 +- 12 files changed, 124 insertions(+), 14 deletions(-) create mode 100644 news/2453.bugfix create mode 100644 news/2453.vendor create mode 100644 pipenv/vendor/shellingham/posix/__init__.py create mode 100644 pipenv/vendor/shellingham/posix/_default.py create mode 100644 pipenv/vendor/shellingham/posix/linux.py diff --git a/news/2453.bugfix b/news/2453.bugfix new file mode 100644 index 00000000..7ea4d825 --- /dev/null +++ b/news/2453.bugfix @@ -0,0 +1,2 @@ +Fixed a bug causing requirements input as relative paths to be output as absolute paths or URIs. +Fixed a bug affecting normalization of ``git+git@host`` uris. diff --git a/news/2453.vendor b/news/2453.vendor new file mode 100644 index 00000000..cea3b695 --- /dev/null +++ b/news/2453.vendor @@ -0,0 +1 @@ +Updated ``requirementslib`` to version ``1.0.8`` diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index 12c3991a..66bfafa6 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -11,7 +11,7 @@ from contextlib import contextmanager from ._compat import InstallRequirement from first import first -from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier +from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier from .click import style diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index ae39b709..88623173 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = "1.0.7" +__version__ = "1.0.8" from .exceptions import RequirementError diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index bd700012..99d8ec76 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -173,6 +173,11 @@ class FileRequirement(BaseRequirement): # This is an URI. We'll need to perform some elaborated parsing. parsed_url = urllib_parse.urlsplit(fixed_line) + if added_ssh_scheme and ':' in parsed_url.netloc: + original_netloc, original_path_start = parsed_url.netloc.rsplit(':', 1) + uri_path = '/{0}{1}'.format(original_path_start, parsed_url.path) + original_url = parsed_url + parsed_url = original_url._replace(netloc=original_netloc, path=uri_path) # Split the VCS part out if needed. original_scheme = parsed_url.scheme @@ -203,7 +208,8 @@ class FileRequirement(BaseRequirement): ) if added_ssh_scheme: - uri = strip_ssh_from_git_uri(uri) + original_uri = urllib_parse.urlunsplit(original_url._replace(scheme=original_scheme, fragment="")) + uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. link = Link( @@ -389,14 +395,14 @@ class FileRequirement(BaseRequirement): @property def line_part(self): - if ( + if self._uri_scheme and self._uri_scheme == 'path': + seed = self.path or unquote(self.link.url_without_fragment) or self.uri + elif ( (self._uri_scheme and self._uri_scheme == "file") - or (self.link.is_artifact or self.link.is_wheel) - and self.link.url + or ((self.link.is_artifact or self.link.is_wheel) + and self.link.url) ): seed = unquote(self.link.url_without_fragment) or self.uri - else: - seed = self.formatted_path or unquote(self.link.url_without_fragment) or self.uri # add egg fragments to remote artifacts (valid urls only) if not self._has_hashed_name and self.is_remote_artifact: seed += "#egg={0}".format(self.name) @@ -528,8 +534,8 @@ class VCSRequirement(FileRequirement): and "git+ssh://" in self.link.url and "git+git@" in self.uri ): - req.line = strip_ssh_from_git_uri(req.line) - req.uri = strip_ssh_from_git_uri(req.uri) + req.line = self.uri + req.uri = self.uri if not req.name: raise ValueError( "pipenv requires an #egg fragment for version controlled " diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index b7daf890..13b2b368 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -84,7 +84,7 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pip format""" + """Cleans VCS uris from pipenv.patched.notpip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: diff --git a/pipenv/vendor/shellingham/__init__.py b/pipenv/vendor/shellingham/__init__.py index 64259c3a..bbdab995 100644 --- a/pipenv/vendor/shellingham/__init__.py +++ b/pipenv/vendor/shellingham/__init__.py @@ -2,7 +2,7 @@ import importlib import os -__version__ = '1.0.0.dev1' +__version__ = '1.1.0' class ShellDetectionFailure(EnvironmentError): diff --git a/pipenv/vendor/shellingham/posix/__init__.py b/pipenv/vendor/shellingham/posix/__init__.py new file mode 100644 index 00000000..ec27b3ac --- /dev/null +++ b/pipenv/vendor/shellingham/posix/__init__.py @@ -0,0 +1,39 @@ +import os +import platform + +from .._consts import SHELL_NAMES + + +def _get_process_mapping(): + system = platform.system() + if system == 'Linux': + from . import linux as impl + else: + from . import _default as impl + return impl.get_process_mapping() + + +def get_shell(pid=None, max_depth=6): + """Get the shell that the supplied pid or os.getpid() is running in. + """ + pid = str(pid or os.getpid()) + mapping = _get_process_mapping() + login_shell = os.environ.get('SHELL', '') + for _ in range(max_depth): + try: + proc = mapping[pid] + except KeyError: + break + name = os.path.basename(proc.args[0]).lower() + if name in SHELL_NAMES: + return (name, proc.args[0]) + elif proc.args[0].startswith('-'): + # This is the login shell. Use the SHELL environ if possible + # because it provides better information. + if login_shell: + name = login_shell.lower() + else: + name = proc.args[0][1:].lower() + return (os.path.basename(name), name) + pid = proc.ppid # Go up one level. + return None diff --git a/pipenv/vendor/shellingham/posix/_default.py b/pipenv/vendor/shellingham/posix/_default.py new file mode 100644 index 00000000..86944276 --- /dev/null +++ b/pipenv/vendor/shellingham/posix/_default.py @@ -0,0 +1,27 @@ +import collections +import shlex +import subprocess +import sys + + +Process = collections.namedtuple('Process', 'args pid ppid') + + +def get_process_mapping(): + """Try to look up the process tree via the output of `ps`. + """ + output = subprocess.check_output([ + 'ps', '-ww', '-o', 'pid=', '-o', 'ppid=', '-o', 'args=', + ]) + if not isinstance(output, str): + output = output.decode(sys.stdout.encoding) + processes = {} + for line in output.split('\n'): + try: + pid, ppid, args = line.strip().split(None, 2) + except ValueError: + continue + processes[pid] = Process( + args=tuple(shlex.split(args)), pid=pid, ppid=ppid, + ) + return processes diff --git a/pipenv/vendor/shellingham/posix/linux.py b/pipenv/vendor/shellingham/posix/linux.py new file mode 100644 index 00000000..6db97834 --- /dev/null +++ b/pipenv/vendor/shellingham/posix/linux.py @@ -0,0 +1,35 @@ +import os +import re + +from ._default import Process + + +STAT_PPID = 3 +STAT_TTY = 6 + + +def get_process_mapping(): + """Try to look up the process tree via Linux's /proc + """ + with open('/proc/{0}/stat'.format(os.getpid())) as f: + self_tty = f.read().split()[STAT_TTY] + processes = {} + for pid in os.listdir('/proc'): + if not pid.isdigit(): + continue + try: + stat = '/proc/{0}/stat'.format(pid) + cmdline = '/proc/{0}/cmdline'.format(pid) + with open(stat) as fstat, open(cmdline) as fcmdline: + stat = re.findall(r'\(.+\)|\S+', fstat.read()) + cmd = fcmdline.read().split('\x00')[:-1] + ppid = stat[STAT_PPID] + tty = stat[STAT_TTY] + if tty == self_tty: + processes[pid] = Process( + args=tuple(cmd), pid=pid, ppid=ppid, + ) + except IOError: + # Process has disappeared - just ignore it. + continue + return processes diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index e95b5178..8ecc6ba5 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -34,7 +34,7 @@ requirementslib==1.0.7 pyparsing==2.2.0 pytoml==0.1.16 requirements-parser==0.2.0 -shellingham==1.1.0dev0 +shellingham==1.1.0 six==1.11.0 semver==2.8.0 shutilwhich==1.1.0 diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index f0e4eddf..d61d7655 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -453,7 +453,7 @@ def license_destination(vendor_dir, libname, filename): normal = vendor_dir / libname if normal.is_dir(): return normal / filename - lowercase = vendor_dir / libname.lower() + lowercase = vendor_dir / libname.lower().replace('-', '_') if lowercase.is_dir(): return lowercase / filename rename_dict = LIBRARY_RENAMES if vendor_dir.name != 'patched' else PATCHED_RENAMES From 3be2badf1f1b58c82ac99557312659850d0baefe Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 27 Jun 2018 21:06:32 -0400 Subject: [PATCH 12/14] Update vendored requirementslib version Signed-off-by: Dan Ryan --- pipenv/vendor/vendor.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 8ecc6ba5..0b2808f0 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -27,7 +27,7 @@ requests==2.19.1 idna==2.7 urllib3==1.23 certifi==2018.4.16 -requirementslib==1.0.7 +requirementslib==1.0.8 attrs==18.1.0 distlib==0.2.7 packaging==17.1 From 1ed07ff0b944b44b743804273384dea0365696c5 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 28 Jun 2018 15:40:43 -0400 Subject: [PATCH 13/14] Update pipfile and setup.py Signed-off-by: Dan Ryan --- Pipfile | 1 + Pipfile.lock | 11 ++++++++++- setup.py | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index ec959c32..f7a07437 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,7 @@ black = {version="*", markers="python_version >= '3.6'"} pytz = "*" towncrier = {git = "https://github.com/hawkowl/towncrier.git", editable = true, ref = "master"} parver = "*" +invoke = "*" [packages] diff --git a/Pipfile.lock b/Pipfile.lock index 4503fff0..48d2873d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -185,6 +185,15 @@ ], "version": "==17.5.0" }, + "invoke": { + "hashes": [ + "sha256:21274204515dca62206470b088bbcf9d41ffda82b3715b90e01d71b7a4681921", + "sha256:4a4cc031db311cbfb3fdd8ec93a06c892533c27b931f4be14b24c97cd042b14e", + "sha256:621b6564f992c37166e16090d7e7cccb3b922e03a58e980dfa5e543a931b652f" + ], + "index": "pypi", + "version": "==1.0.0" + }, "itsdangerous": { "hashes": [ "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" @@ -480,4 +489,4 @@ "version": "==0.14.1" } } -} +} \ No newline at end of file diff --git a/setup.py b/setup.py index 2cbe33ff..a2a29c3e 100644 --- a/setup.py +++ b/setup.py @@ -123,6 +123,7 @@ setup( "pipenv.patched.notpip._vendor.distlib": ["t32.exe", "t64.exe", "w32.exe", "w64.exe"], }, python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*', + setup_requires=['invoke', 'parver'], install_requires=required, extras_require={}, include_package_data=True, From b33356012841ad9c41878cf29a75e58307b336e7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 28 Jun 2018 16:38:08 -0400 Subject: [PATCH 14/14] Update lockfile Signed-off-by: Dan Ryan --- Pipfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 48d2873d..21b32705 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ee9e8c04f9a6b7ad4cdaa7db728b51b9fef04b6cad9f012edd230a6241b43f92" + "sha256": "0599ae763bc5f1eaa30bd414c1495dc10fdfd3b5b0c109515d71322c8668d7e1" }, "pipfile-spec": 6, "requires": {}, @@ -489,4 +489,4 @@ "version": "==0.14.1" } } -} \ No newline at end of file +}