From bc3d50b8bec452c57b3dc4d3c204360d91babe09 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 26 Feb 2019 12:03:26 -0500 Subject: [PATCH 01/21] Update vendored vistir Signed-off-by: Dan Ryan --- pipenv/vendor/vistir/__init__.py | 7 ++- pipenv/vendor/vistir/cursor.py | 76 ++++++++++++++++++++++++++++++++ pipenv/vendor/vistir/spin.py | 7 ++- 3 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 pipenv/vendor/vistir/cursor.py diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index 9bb369bb..ecc67c1a 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -17,6 +17,7 @@ from .contextmanagers import ( spinner, replaced_stream ) +from .cursor import show_cursor, hide_cursor from .misc import ( load_path, partialclass, @@ -35,7 +36,7 @@ from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfi from .spin import create_spinner -__version__ = '0.3.0' +__version__ = '0.3.1' __all__ = [ @@ -67,5 +68,7 @@ __all__ = [ "StringIO", "get_wrapped_stream", "StreamWrapper", - "replaced_stream" + "replaced_stream", + "show_cursor", + "hide_cursor" ] diff --git a/pipenv/vendor/vistir/cursor.py b/pipenv/vendor/vistir/cursor.py new file mode 100644 index 00000000..ce998f51 --- /dev/null +++ b/pipenv/vendor/vistir/cursor.py @@ -0,0 +1,76 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import ctypes +import os +import sys + + +__all__ = ["hide_cursor", "show_cursor"] + + +class CONSOLE_CURSOR_INFO(ctypes.Structure): + _fields_ = [('dwSize', ctypes.c_int), + ('bVisible', ctypes.c_int)] + + +WIN_STDERR_HANDLE_ID = ctypes.c_ulong(-12) +WIN_STDOUT_HANDLE_ID = ctypes.c_ulong(-11) + + +def get_stream_handle(stream=sys.stdout): + """ + Get the OS appropriate handle for the corresponding output stream. + + :param str stream: The the stream to get the handle for + :return: A handle to the appropriate stream, either a ctypes buffer + or **sys.stdout** or **sys.stderr**. + """ + handle = stream + if os.name == "nt": + from ctypes import windll + handle_id = WIN_STDOUT_HANDLE_ID + handle = windll.kernel32.GetStdHandle(handle_id) + return handle + + +def hide_cursor(stream=sys.stdout): + """ + Hide the console cursor on the given stream + + :param stream: The name of the stream to get the handle for + :return: None + :rtype: None + """ + + handle = get_stream_handle(stream=stream) + if os.name == "nt": + from ctypes import windll + cursor_info = CONSOLE_CURSOR_INFO() + windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(cursor_info)) + cursor_info.visible = False + windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(cursor_info)) + else: + handle.write("\033[?25l") + handle.flush() + + +def show_cursor(stream=sys.stdout): + """ + Show the console cursor on the given stream + + :param stream: The name of the stream to get the handle for + :return: None + :rtype: None + """ + + handle = get_stream_handle(stream=stream) + if os.name == "nt": + from ctypes import windll + cursor_info = CONSOLE_CURSOR_INFO() + windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(cursor_info)) + cursor_info.visible = True + windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(cursor_info)) + else: + handle.write("\033[?25h") + handle.flush() diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index a9275055..81ae3014 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -12,18 +12,17 @@ import colorama import six from .compat import to_native_string +from .cursor import hide_cursor, show_cursor from .termcolors import COLOR_MAP, COLORS, colored, DISABLE_COLORS from .misc import decode_for_output from io import StringIO try: import yaspin - import cursor except ImportError: yaspin = None Spinners = None SpinBase = None - cursor = None else: import yaspin.spinners import yaspin.core @@ -421,13 +420,13 @@ class VistirSpinner(SpinBase): def _hide_cursor(target=None): if not target: target = sys.stdout - cursor.hide(stream=target) + hide_cursor(stream=target) @staticmethod def _show_cursor(target=None): if not target: target = sys.stdout - cursor.show(stream=target) + show_cursor(stream=target) @staticmethod def _clear_err(): From 32876ecc90b6731cabccbad236b4503f632bf7ca Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 26 Feb 2019 17:36:02 -0500 Subject: [PATCH 02/21] Update requirementslib Signed-off-by: Dan Ryan --- .../vendor/requirementslib/models/pipfile.py | 12 ++-- .../requirementslib/models/requirements.py | 58 ++++++++++++------- .../requirementslib/models/setup_info.py | 12 ++-- pipenv/vendor/requirementslib/models/utils.py | 10 ++-- 4 files changed, 57 insertions(+), 35 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 021b5d53..22eee048 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -22,7 +22,7 @@ from .utils import optional_instance_of, get_url_name from ..environment import MYPY_RUNNING if MYPY_RUNNING: - from typing import Union, Any, Dict, Iterable, Sequence, Mapping, List, NoReturn, Text + from typing import Union, Any, Dict, Iterable, Mapping, List, Text package_type = Dict[Text, Dict[Text, Union[List[Text], Text]]] source_type = Dict[Text, Union[Text, bool]] sources_type = Iterable[source_type] @@ -264,7 +264,8 @@ class Pipfile(object): @classmethod def load_projectfile(cls, path, create=False): # type: (Text, bool) -> ProjectFile - """Given a path, load or create the necessary pipfile. + """ + Given a path, load or create the necessary pipfile. :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True @@ -289,7 +290,8 @@ class Pipfile(object): @classmethod def load(cls, path, create=False): # type: (Text, bool) -> Pipfile - """Given a path, load or create the necessary pipfile. + """ + Given a path, load or create the necessary pipfile. :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True @@ -340,8 +342,8 @@ class Pipfile(object): if not os.path.exists(self.path_to("setup.py")): if not build_system or not build_system.get("requires"): build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8", "wheel"], + "build-backend": "setuptools.build_meta:__legacy__", } self._build_system = build_system diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index d6c13819..67065f85 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -77,7 +77,7 @@ from .utils import ( from ..environment import MYPY_RUNNING if MYPY_RUNNING: - from typing import Optional, TypeVar, List, Dict, Union, Any, Tuple, Generator, Set, Text + from typing import Optional, TypeVar, List, Dict, Union, Any, Tuple, Set, Text from pip_shims.shims import Link, InstallRequirement RequirementType = TypeVar('RequirementType', covariant=True, bound=PackagingRequirement) from six.moves.urllib.parse import SplitResult @@ -320,8 +320,8 @@ class Line(object): @specifiers.setter def specifiers(self, specifiers): # type: (Union[Text, SpecifierSet]) -> None - if type(specifiers) is not SpecifierSet: - if type(specifiers) in six.string_types: + if not isinstance(specifiers, SpecifierSet): + if isinstance(specifiers, six.string_types): specifiers = SpecifierSet(specifiers) else: raise TypeError("Must pass a string or a SpecifierSet") @@ -739,7 +739,7 @@ class Line(object): wheel_kwargs = self.wheel_kwargs.copy() wheel_kwargs["src_dir"] = repo.checkout_directory ireq.source_dir = wheel_kwargs["src_dir"] - build_dir = ireq.build_location(wheel_kwargs["build_dir"]) + ireq.build_location(wheel_kwargs["build_dir"]) ireq._temp_build_dir.path = wheel_kwargs["build_dir"] with temp_path(): sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)] @@ -936,6 +936,8 @@ class Line(object): self.relpath = relpath self.path = path self.uri = uri + if prefer in ("path", "relpath") or uri.startswith("file"): + self.is_local = True if link.egg_fragment: name, extras = pip_shims.shims._strip_extras(link.egg_fragment) self.extras = tuple(sorted(set(parse_extras(extras)))) @@ -1787,7 +1789,7 @@ class FileRequirement(object): line = "{0}{1}".format(line, extras_to_string(extras)) if "subdirectory" in pipfile: arg_dict["subdirectory"] = pipfile["subdirectory"] - line = "{0}&subdirectory={1}".format(pipfile["subdirectory"]) + line = "{0}&subdirectory={1}".format(line, pipfile["subdirectory"]) if pipfile.get("editable", False): line = "-e {0}".format(line) arg_dict["line"] = line @@ -2520,7 +2522,8 @@ class Requirement(object): @property def build_backend(self): # type: () -> Optional[Text] - if self.is_vcs or (self.is_file_or_url and self.req.is_local): + if self.is_vcs or (self.is_file_or_url and ( + self.req is not None and self.req.is_local)): setup_info = self.run_requires() build_backend = setup_info.get("build_backend") return build_backend @@ -2546,11 +2549,15 @@ class Requirement(object): @property def is_wheel(self): # type: () -> bool - if not self.is_named and self.req.link is not None and self.req.link.is_wheel: + if not self.is_named and ( + self.req is not None and + self.req.link is not None and + self.req.link.is_wheel + ): return True return False - @cached_property + @property def normalized_name(self): # type: () -> Text return canonicalize_name(self.name) @@ -2600,7 +2607,7 @@ class Requirement(object): r.req.extras = args["extras"] if parsed_line.hashes: args["hashes"] = tuple(parsed_line.hashes) # type: ignore - cls_inst = cls(**args) + cls_inst = cls(**args) # type: ignore return cls_inst @classmethod @@ -2694,7 +2701,9 @@ class Requirement(object): parts.extend(hashes) else: parts.append(hashes) - if sources and not (self.requirement.local_file or self.vcs): + + is_local = self.is_file_or_url and self.req and self.req.is_local + if sources and self.requirement and not (is_local or self.vcs): from ..utils import prepare_pip_source_args if self.index: @@ -2938,6 +2947,9 @@ class Requirement(object): def file_req_from_parsed_line(parsed_line): # type: (Line) -> FileRequirement path = parsed_line.relpath if parsed_line.relpath else parsed_line.path + pyproject_requires = () # type: Tuple[Text] + if parsed_line.pyproject_requires is not None: + pyproject_requires = tuple(parsed_line.pyproject_requires) return FileRequirement( setup_path=parsed_line.setup_py, path=path, @@ -2946,7 +2958,7 @@ def file_req_from_parsed_line(parsed_line): uri_scheme=parsed_line.preferred_scheme, link=parsed_line.link, uri=parsed_line.uri, - pyproject_requires=tuple(parsed_line.pyproject_requires) if parsed_line.pyproject_requires else None, + pyproject_requires=pyproject_requires, pyproject_backend=parsed_line.pyproject_backend, pyproject_path=Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None, parsed_line=parsed_line, @@ -2960,14 +2972,20 @@ def vcs_req_from_parsed_line(parsed_line): line = "{0}".format(parsed_line.line) if parsed_line.editable: line = "-e {0}".format(line) - link = create_link(build_vcs_uri( - vcs=parsed_line.vcs, - uri=parsed_line.url, - name=parsed_line.name, - ref=parsed_line.ref, - subdirectory=parsed_line.subdirectory, - extras=parsed_line.extras - )) + if parsed_line.url is not None: + link = create_link(build_vcs_uri( + vcs=parsed_line.vcs, + uri=parsed_line.url, + name=parsed_line.name, + ref=parsed_line.ref, + subdirectory=parsed_line.subdirectory, + extras=list(parsed_line.extras) + )) + else: + link = parsed_line.link + pyproject_requires = () # type: Tuple[Text] + if parsed_line.pyproject_requires is not None: + pyproject_requires = tuple(parsed_line.pyproject_requires) return VCSRequirement( setup_path=parsed_line.setup_py, path=parsed_line.path, @@ -2979,7 +2997,7 @@ def vcs_req_from_parsed_line(parsed_line): uri_scheme=parsed_line.preferred_scheme, link=link, uri=parsed_line.uri, - pyproject_requires=tuple(parsed_line.pyproject_requires) if parsed_line.pyproject_requires else None, + pyproject_requires=pyproject_requires, pyproject_backend=parsed_line.pyproject_backend, pyproject_path=Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None, parsed_line=parsed_line, diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index b0b55d47..e599f6f7 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -15,7 +15,6 @@ import pep517.envbuild import pep517.wrappers import six from appdirs import user_cache_dir -from cached_property import cached_property from distlib.wheel import Wheel from packaging.markers import Marker from six.moves import configparser @@ -303,8 +302,8 @@ def get_metadata_from_wheel(wheel_path): name = metadata.name version = metadata.version requires = [] - extras_keys = getattr(metadata, "extras", None) - extras = {} + extras_keys = getattr(metadata, "extras", []) + extras = {k: [] for k in extras_keys} for req in getattr(metadata, "run_requires", []): parsed_req = init_requirement(req) parsed_marker = parsed_req.marker @@ -652,6 +651,7 @@ build-backend = "{1}" dist_type="wheel" ) + # noinspection PyPackageRequirements def build_sdist(self): # type: () -> Text if not self.pyproject.exists(): @@ -718,7 +718,7 @@ build-backend = "{1}" get_metadata(d, pkg_name=self.name, metadata_type=metadata_type) for d in metadata_dirs if os.path.exists(d) ] - metadata = next(iter(d for d in metadata if d is not None), None) + metadata = next(iter(d for d in metadata if d), None) if metadata is not None: self.populate_metadata(metadata) @@ -839,7 +839,7 @@ build-backend = "{1}" from .dependencies import get_finder finder = get_finder() - vcs_method, uri = split_vcs_method_from_uri(unquote(ireq.link.url_without_fragment)) + _, uri = split_vcs_method_from_uri(unquote(ireq.link.url_without_fragment)) parsed = urlparse(uri) if "file" in parsed.scheme: url_path = parsed.path @@ -870,7 +870,7 @@ build-backend = "{1}" "The file URL points to a directory not installable: {}" .format(ireq.link) ) - build_dir = ireq.build_location(kwargs["build_dir"]) + ireq.build_location(kwargs["build_dir"]) src_dir = ireq.ensure_has_source_dir(kwargs["src_dir"]) ireq._temp_build_dir.path = kwargs["build_dir"] diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 8ce441e2..387997f3 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -295,9 +295,11 @@ def strip_extras_markers_from_requirement(req): raise TypeError("Must pass in a valid requirement, received {0!r}".format(req)) if getattr(req, "marker", None) is not None: marker = req.marker # type: TMarker - req.marker._markers = _strip_extras_markers(req.marker._markers) - if not req.marker._markers: + marker._markers = _strip_extras_markers(marker._markers) + if not marker._markers: req.marker = None + else: + req.marker = marker return req @@ -338,7 +340,7 @@ def get_default_pyproject_backend(): st_version = get_setuptools_version() if st_version is not None: parsed_st_version = parse_version(st_version) - if parsed_st_version >= parse_version("40.6.0"): + if parsed_st_version >= parse_version("40.8.0"): return "setuptools.build_meta:__legacy__" return "setuptools.build_meta" @@ -354,9 +356,9 @@ def get_pyproject(path): :rtype: Tuple[List[Text], Text] """ - from vistir.compat import Path if not path: return + from vistir.compat import Path if not isinstance(path, Path): path = Path(path) if not path.is_dir(): From 3f053a355f0a6936939542ae791c8c6c2aec9424 Mon Sep 17 00:00:00 2001 From: frostming Date: Wed, 27 Feb 2019 12:21:23 +0800 Subject: [PATCH 03/21] Fix command dispatching --- pipenv/cli/command.py | 8 +------- pipenv/cli/options.py | 3 ++- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 16ce115e..f1fac2b2 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -12,8 +12,6 @@ import click_completion import crayons import delegator -from click_didyoumean import DYMCommandCollection - from ..__version__ import __version__ from .options import ( CONTEXT_SETTINGS, PipenvGroup, code_option, common_options, deploy_option, @@ -400,8 +398,7 @@ def shell( @pass_state def run(state, command, args): """Spawns a command installed into the virtualenv.""" - from ..core import do_run, warn_in_virtualenv - warn_in_virtualenv() + from ..core import do_run do_run( command=command, args=args, three=state.three, python=state.python, pypi_mirror=state.pypi_mirror ) @@ -634,8 +631,5 @@ def clean(ctx, state, dry_run=False, bare=False, user=False): system=state.system) -# Only invoke the "did you mean" when an argument wasn't passed (it breaks those). -if "-" not in "".join(sys.argv) and len(sys.argv) > 1: - cli = DYMCommandCollection(sources=[cli]) if __name__ == "__main__": cli() diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 745275dd..8ca38ae5 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -8,6 +8,7 @@ import click.types from click import ( BadParameter, Group, Option, argument, echo, make_pass_decorator, option ) +from click_didyoumean import DYMMixin from .. import environments from ..utils import is_valid_url @@ -19,7 +20,7 @@ CONTEXT_SETTINGS = { } -class PipenvGroup(Group): +class PipenvGroup(DYMMixin, Group): """Custom Group class provides formatted main help""" def get_help_option(self, ctx): From 4848a1e5079025a3c8a3502b840f3b678d097066 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 1 Mar 2019 17:01:29 -0500 Subject: [PATCH 04/21] Update vistir and requirementslib Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/__init__.py | 2 +- .../requirementslib/models/requirements.py | 991 ++++++++++-------- .../requirementslib/models/setup_info.py | 89 +- pipenv/vendor/requirementslib/models/utils.py | 98 +- pipenv/vendor/vistir/cmdparse.py | 16 +- 5 files changed, 658 insertions(+), 538 deletions(-) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index c7156971..8246f036 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,6 +1,6 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function -__version__ = '1.4.1.dev0' +__version__ = '1.4.1' import logging import warnings diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 67065f85..d6beb085 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -25,7 +25,7 @@ from packaging.specifiers import Specifier, SpecifierSet, LegacySpecifier, Inval from packaging.utils import canonicalize_name from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote -from vistir.compat import Path, FileNotFoundError, lru_cache +from vistir.compat import Path, FileNotFoundError, lru_cache, Mapping from vistir.contextmanagers import temp_path from vistir.misc import dedup from vistir.path import ( @@ -77,12 +77,19 @@ from .utils import ( from ..environment import MYPY_RUNNING if MYPY_RUNNING: - from typing import Optional, TypeVar, List, Dict, Union, Any, Tuple, Set, Text - from pip_shims.shims import Link, InstallRequirement + from typing import Optional, TypeVar, List, Dict, Union, Any, Tuple, Set, AnyStr, Text, Generator, FrozenSet + from pip_shims.shims import Link, InstallRequirement, PackageFinder, InstallationCandidate RequirementType = TypeVar('RequirementType', covariant=True, bound=PackagingRequirement) + F = TypeVar("F", "FileRequirement", "VCSRequirement", covariant=True) from six.moves.urllib.parse import SplitResult from .vcs import VCSRepository + from .dependencies import AbstractDependency NON_STRING_ITERABLE = Union[List, Set, Tuple] + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) + BASE_TYPES = Union[bool, STRING_TYPE, Tuple[STRING_TYPE, ...]] + CUSTOM_TYPES = Union[VCSRepository, RequirementType, SetupInfo, "Line"] + CREATION_ARG_TYPES = Union[BASE_TYPES, Link, CUSTOM_TYPES] SPECIFIERS_BY_LENGTH = sorted(list(Specifier._operators.keys()), key=len, reverse=True) @@ -93,41 +100,41 @@ run = partial(vistir.misc.run, combine_stderr=False, return_object=True, nospin= class Line(object): def __init__(self, line, extras=None): - # type: (Text, Optional[NON_STRING_ITERABLE]) -> None + # type: (AnyStr, Optional[Union[List[S], Set[S], Tuple[S, ...]]]) -> None self.editable = False # type: bool if line.startswith("-e "): line = line[len("-e "):] self.editable = True - self.extras = () # type: Tuple[Text] + self.extras = () # type: Tuple[STRING_TYPE, ...] if extras is not None: self.extras = tuple(sorted(set(extras))) - self.line = line # type: Text - self.hashes = [] # type: List[Text] - self.markers = None # type: Optional[Text] - self.vcs = None # type: Optional[Text] - self.path = None # type: Optional[Text] - self.relpath = None # type: Optional[Text] - self.uri = None # type: Optional[Text] + self.line = line # type: STRING_TYPE + self.hashes = [] # type: List[STRING_TYPE] + self.markers = None # type: Optional[STRING_TYPE] + self.vcs = None # type: Optional[STRING_TYPE] + self.path = None # type: Optional[STRING_TYPE] + self.relpath = None # type: Optional[STRING_TYPE] + self.uri = None # type: Optional[STRING_TYPE] self._link = None # type: Optional[Link] self.is_local = False # type: bool - self._name = None # type: Optional[Text] - self._specifier = None # type: Optional[Text] + self._name = None # type: Optional[STRING_TYPE] + self._specifier = None # type: Optional[STRING_TYPE] self.parsed_marker = None # type: Optional[Marker] - self.preferred_scheme = None # type: Optional[Text] + self.preferred_scheme = None # type: Optional[STRING_TYPE] self._requirement = None # type: Optional[PackagingRequirement] self.is_direct_url = False # type: bool self._parsed_url = None # type: Optional[urllib_parse.ParseResult] - self._setup_cfg = None # type: Optional[Text] - self._setup_py = None # type: Optional[Text] - self._pyproject_toml = None # type: Optional[Text] - self._pyproject_requires = None # type: Optional[List[Text]] - self._pyproject_backend = None # type: Optional[Text] - self._wheel_kwargs = None # type: Dict[Text, Text] + self._setup_cfg = None # type: Optional[STRING_TYPE] + self._setup_py = None # type: Optional[STRING_TYPE] + self._pyproject_toml = None # type: Optional[STRING_TYPE] + self._pyproject_requires = None # type: Optional[Tuple[STRING_TYPE, ...]] + self._pyproject_backend = None # type: Optional[STRING_TYPE] + self._wheel_kwargs = None # type: Optional[Dict[STRING_TYPE, STRING_TYPE]] self._vcsrepo = None # type: Optional[VCSRepository] self._setup_info = None # type: Optional[SetupInfo] - self._ref = None # type: Optional[Text] + self._ref = None # type: Optional[STRING_TYPE] self._ireq = None # type: Optional[InstallRequirement] - self._src_root = None # type: Optional[Text] + self._src_root = None # type: Optional[STRING_TYPE] self.dist = None # type: Any super(Line, self).__init__() self.parse() @@ -147,18 +154,19 @@ class Line(object): "pyproject_requires={self._pyproject_requires}, " "pyproject_backend={self._pyproject_backend}, ireq={self._ireq})>".format( self=self - )) + ) + ) except Exception: return "".format(self.__dict__.values()) @classmethod def split_hashes(cls, line): - # type: (Text) -> Tuple[Text, List[Text]] + # type: (S) -> Tuple[S, List[S]] if "--hash" not in line: return line, [] split_line = line.split() - line_parts = [] # type: List[Text] - hashes = [] # type: List[Text] + line_parts = [] # type: List[S] + hashes = [] # type: List[S] for part in split_line: if part.startswith("--hash"): param, _, value = part.partition("=") @@ -170,13 +178,11 @@ class Line(object): @property def line_with_prefix(self): - # type: () -> Text + # type: () -> STRING_TYPE line = self.line extras_str = extras_to_string(self.extras) if self.is_direct_url: line = self.link.url - # if self.link.egg_info and self.extras: - # line = "{0}{1}".format(line, extras_str) elif extras_str: if self.is_vcs: line = self.link.url @@ -190,8 +196,8 @@ class Line(object): @property def line_for_ireq(self): - # type: () -> Text - line = "" + # type: () -> STRING_TYPE + line = "" # type: STRING_TYPE if self.is_file or self.is_url and not self.is_vcs: scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri" local_line = next(iter([ @@ -231,7 +237,7 @@ class Line(object): @property def base_path(self): - # type: () -> Optional[Text] + # type: () -> Optional[S] if not self.link and not self.path: self.parse_link() if not self.path: @@ -247,40 +253,42 @@ class Line(object): @property def setup_py(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self._setup_py is None: self.populate_setup_paths() return self._setup_py @property def setup_cfg(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self._setup_cfg is None: self.populate_setup_paths() return self._setup_cfg @property def pyproject_toml(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self._pyproject_toml is None: self.populate_setup_paths() return self._pyproject_toml @property def specifier(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] options = [self._specifier] for req in (self.ireq, self.requirement): if req is not None and getattr(req, "specifier", None): options.append(req.specifier) - specifier = next(iter(spec for spec in options if spec is not None), None) + specifier = next(iter(spec for spec in options if spec is not None), None) # type: Optional[Union[Specifier, SpecifierSet]] + spec_string = None # type: Optional[STRING_TYPE] if specifier is not None: - specifier = specs_to_string(specifier) - elif specifier is None and not self.is_named and self._setup_info is not None: - if self._setup_info.version: - specifier = "=={0}".format(self._setup_info.version) - if specifier: - self._specifier = specifier + spec_string = specs_to_string(specifier) + elif specifier is None and not self.is_named and ( + self._setup_info is not None and self._setup_info.version + ): + spec_string = "=={0}".format(self._setup_info.version) + if spec_string: + self._specifier = spec_string return self._specifier @specifier.setter @@ -319,14 +327,14 @@ class Line(object): @specifiers.setter def specifiers(self, specifiers): - # type: (Union[Text, SpecifierSet]) -> None + # type: (Union[Text, str, SpecifierSet]) -> None if not isinstance(specifiers, SpecifierSet): if isinstance(specifiers, six.string_types): specifiers = SpecifierSet(specifiers) else: raise TypeError("Must pass a string or a SpecifierSet") specs = self.get_requirement_specs(specifiers) - if self.ireq is not None and self.ireq.req is not None: + if self.ireq is not None and self._ireq and self._ireq.req is not None: self._ireq.req.specifier = specifiers self._ireq.req.specs = specs if self.requirement is not None: @@ -335,7 +343,7 @@ class Line(object): @classmethod def get_requirement_specs(cls, specifierset): - # type: (SpecifierSet) -> List[Tuple[Text, Text]] + # type: (SpecifierSet) -> List[Tuple[AnyStr, AnyStr]] specs = [] spec = next(iter(specifierset._specs), None) if spec: @@ -344,7 +352,7 @@ class Line(object): @property def requirement(self): - # type: () -> Optional[PackagingRequirement] + # type: () -> Optional[RequirementType] if self._requirement is None: self.parse_requirement() if self._requirement is None and self._name is not None: @@ -365,31 +373,32 @@ class Line(object): base_path = self.base_path if base_path is None: return - setup_paths = get_setup_paths(self.base_path, subdirectory=self.subdirectory) # type: Dict[Text, Optional[Text]] + setup_paths = get_setup_paths(base_path, subdirectory=self.subdirectory) # type: Dict[STRING_TYPE, Optional[STRING_TYPE]] self._setup_py = setup_paths.get("setup_py") self._setup_cfg = setup_paths.get("setup_cfg") self._pyproject_toml = setup_paths.get("pyproject_toml") @property def pyproject_requires(self): - # type: () -> Optional[List[Text]] + # type: () -> Optional[Tuple[STRING_TYPE, ...]] if self._pyproject_requires is None and self.pyproject_toml is not None: - pyproject_requires, pyproject_backend = get_pyproject(self.path) - self._pyproject_requires = pyproject_requires + pyproject_requires, pyproject_backend = get_pyproject(self.path) # type: ignore + if pyproject_requires: + self._pyproject_requires = tuple(pyproject_requires) self._pyproject_backend = pyproject_backend return self._pyproject_requires @property def pyproject_backend(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self._pyproject_requires is None and self.pyproject_toml is not None: - pyproject_requires, pyproject_backend = get_pyproject(self.path) + pyproject_requires, pyproject_backend = get_pyproject(self.path) # type: ignore if not pyproject_backend and self.setup_cfg is not None: setup_dict = SetupInfo.get_setup_cfg(self.setup_cfg) pyproject_backend = get_default_pyproject_backend() - pyproject_requires = setup_dict.get("build_requires", ["setuptools", "wheel"]) + pyproject_requires = setup_dict.get("build_requires", ["setuptools", "wheel"]) # type: ignore - self._pyproject_requires = pyproject_requires + self._pyproject_requires = tuple(pyproject_requires) self._pyproject_backend = pyproject_backend return self._pyproject_backend @@ -414,6 +423,7 @@ class Line(object): """ extras = None + url = "" # type: STRING_TYPE if "@" in self.line or self.is_vcs or self.is_url: line = "{0}".format(self.line) match = DIRECT_URL_RE.match(line) @@ -431,7 +441,8 @@ class Line(object): ref = match_dict.get("ref") subdir = match_dict.get("subdirectory") pathsep = match_dict.get("pathsep", "/") - url = scheme + if scheme is not None: + url = scheme if host: url = "{0}{1}".format(url, host) if path: @@ -451,49 +462,23 @@ class Line(object): self.line = add_ssh_scheme_to_git_uri(url) if name: self._name = name - # line = add_ssh_scheme_to_git_uri(self.line) - # parsed = urllib_parse.urlparse(line) - # if not parsed.scheme and "@" in line: - # matched = URL_RE.match(line) - # if matched is None: - # matched = NAME_RE.match(line) - # if matched: - # name = matched.groupdict().get("name") - # if name is not None: - # self._name = name - # extras = matched.groupdict().get("extras") - # else: - # name, _, line = self.line.partition("@") - # name = name.strip() - # line = line.strip() - # matched = NAME_RE.match(name) - # match_dict = matched.groupdict() - # name = match_dict.get("name") - # extras = match_dict.get("extras") - # if is_vcs(line) or is_valid_url(line): - # self.is_direct_url = True - # # name, extras = pip_shims.shims._strip_extras(name) - # self._name = name - # self.line = line else: self.line, extras = pip_shims.shims._strip_extras(self.line) else: self.line, extras = pip_shims.shims._strip_extras(self.line) + extras_set = set() # type: Set[STRING_TYPE] if extras is not None: - extras = set(parse_extras(extras)) + extras_set = set(parse_extras(extras)) if self._name: self._name, name_extras = pip_shims.shims._strip_extras(self._name) if name_extras: name_extras = set(parse_extras(name_extras)) - if extras: - extras |= name_extras - else: - extras = name_extras - if extras is not None: - self.extras = tuple(sorted(extras)) + extras_set |= name_extras + if extras_set is not None: + self.extras = tuple(sorted(extras_set)) def get_url(self): - # type: () -> Text + # type: () -> STRING_TYPE """Sets ``self.name`` if given a **PEP-508** style URL""" line = self.line @@ -521,7 +506,7 @@ class Line(object): @property def name(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self._name is None: self.parse_name() if self._name is None and not self.is_named and not self.is_wheel: @@ -531,18 +516,18 @@ class Line(object): @name.setter def name(self, name): - # type: (Text) -> None + # type: (STRING_TYPE) -> None self._name = name if self._setup_info: self._setup_info.name = name - if self.requirement: + if self.requirement and self._requirement: self._requirement.name = name - if self.ireq and self.ireq.req: + if self.ireq and self._ireq and self._ireq.req: self._ireq.req.name = name @property def url(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self.uri is not None: url = add_ssh_scheme_to_git_uri(self.uri) else: @@ -567,7 +552,7 @@ class Line(object): @property def subdirectory(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self.link is not None: return self.link.subdirectory_fragment return "" @@ -645,7 +630,7 @@ class Line(object): @property def ref(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self._ref is None and self.relpath is not None: self.relpath, self._ref = split_ref_from_uri(self.relpath) return self._ref @@ -778,7 +763,7 @@ class Line(object): self._ireq.req = self.requirement def _parse_wheel(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if not self.is_wheel: pass from pip_shims.shims import Wheel @@ -789,7 +774,7 @@ class Line(object): return name def _parse_name_from_link(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self.link is None: return None @@ -800,8 +785,7 @@ class Line(object): return None def _parse_name_from_line(self): - # type: () -> Optional[Text] - + # type: () -> Optional[STRING_TYPE] if not self.is_named: pass try: @@ -818,9 +802,13 @@ class Line(object): specifier_match = next( iter(spec for spec in SPECIFIERS_BY_LENGTH if spec in self.line), None ) - if specifier_match is not None: - name, specifier_match, version = name.partition(specifier_match) - self._specifier = "{0}{1}".format(specifier_match, version) + specifier = None # type: Optional[STRING_TYPE] + if specifier_match: + specifier = "{0!s}".format(specifier_match) + if specifier is not None and specifier in name: + version = None # type: Optional[STRING_TYPE] + name, specifier, version = name.partition(specifier) + self._specifier = "{0}{1}".format(specifier, version) return name def parse_name(self): @@ -847,24 +835,33 @@ class Line(object): def _parse_requirement_from_vcs(self): # type: () -> Optional[PackagingRequirement] + url = self.url if self.url else self.link.url + if url: + url = unquote(url) if ( - self.uri != unquote(self.url) - and "git+ssh://" in self.url + url + and self.uri != url + and "git+ssh://" in url and (self.uri is not None and "git+git@" in self.uri) + and self._requirement is not None ): self._requirement.line = self.uri self._requirement.url = self.url - self._requirement.link = create_link(build_vcs_uri( + vcs_uri = build_vcs_uri( # type: ignore vcs=self.vcs, uri=self.url, ref=self.ref, subdirectory=self.subdirectory, extras=self.extras, name=self.name - )) + ) + if vcs_uri: + self._requirement.link = create_link(vcs_uri) + elif self.link: + self._requirement.link = self.link # else: # req.link = self.link - if self.ref: + if self.ref and self._requirement is not None: if self._vcsrepo is not None: self._requirement.revision = self._vcsrepo.get_commit_hash() else: @@ -963,20 +960,20 @@ class Line(object): @property def requirement_info(self): - # type: () -> Tuple(Optional[Text], Tuple[Optional[Text]], Optional[Text]) + # type: () -> Tuple[Optional[S], Tuple[Optional[S], ...], Optional[S]] """ Generates a 3-tuple of the requisite *name*, *extras* and *url* to generate a :class:`~packaging.requirements.Requirement` out of. :return: A Tuple containing an optional name, a Tuple of extras names, and an optional URL. - :rtype: Tuple[Optional[Text], Tuple[Optional[Text]], Optional[Text]] + :rtype: Tuple[Optional[S], Tuple[Optional[S], ...], Optional[S]] """ # Direct URLs can be converted to packaging requirements directly, but # only if they are `file://` (with only two slashes) - name = None - extras = () - url = None + name = None # type: Optional[S] + extras = () # type: Tuple[Optional[S], ...] + url = None # type: Optional[STRING_TYPE] # if self.is_direct_url: if self._name: name = canonicalize_name(self._name) @@ -999,10 +996,7 @@ class Line(object): self._name = self.link.egg_fragment if self._name: name = canonicalize_name(self._name) - # return "{0}{1}@ {2}".format( - # normalize_name(self.name), extras_to_string(self.extras), url - # ) - return (name, extras, url) + return name, extras, url # type: ignore @property def line_is_installable(self): @@ -1050,10 +1044,10 @@ class Line(object): @attr.s(slots=True, hash=True) class NamedRequirement(object): - name = attr.ib() # type: Text - version = attr.ib() # type: Optional[Text] + name = attr.ib() # type: STRING_TYPE + version = attr.ib() # type: Optional[STRING_TYPE] req = attr.ib() # type: PackagingRequirement - extras = attr.ib(default=attr.Factory(list)) # type: Tuple[Text] + extras = attr.ib(default=attr.Factory(list)) # type: Tuple[STRING_TYPE, ...] editable = attr.ib(default=False) # type: bool _parsed_line = attr.ib(default=None) # type: Optional[Line] @@ -1074,9 +1068,9 @@ class NamedRequirement(object): @classmethod def from_line(cls, line, parsed_line=None): - # type: (Text, Optional[Line]) -> NamedRequirement + # type: (AnyStr, Optional[Line]) -> NamedRequirement req = init_requirement(line) - specifiers = None # type: Optional[Text] + specifiers = None # type: Optional[STRING_TYPE] if req.specifier: specifiers = specs_to_string(req.specifier) req.line = line @@ -1094,21 +1088,21 @@ class NamedRequirement(object): "parsed_line": parsed_line, "extras": None } - extras = None # type: Optional[Tuple[Text]] + extras = None # type: Optional[Tuple[STRING_TYPE, ...]] if req.extras: - extras = list(req.extras) + extras = tuple(req.extras) creation_kwargs["extras"] = extras return cls(**creation_kwargs) @classmethod def from_pipfile(cls, name, pipfile): - # type: (Text, Dict[Text, Union[Text, Optional[Text], Optional[List[Text]]]]) -> NamedRequirement - creation_args = {} # type: Dict[Text, Union[Optional[Text], Optional[List[Text]]]] + # type: (S, Dict[S, Union[S, bool, Union[List[S], Tuple[S, ...], Set[S]]]]) -> NamedRequirement + creation_args = {} # type: Dict[STRING_TYPE, Union[Optional[STRING_TYPE], Optional[List[STRING_TYPE]]]] if hasattr(pipfile, "keys"): attr_fields = [field.name for field in attr.fields(cls)] - creation_args = {k: v for k, v in pipfile.items() if k in attr_fields} + creation_args = {k: v for k, v in pipfile.items() if k in attr_fields} # type: ignore creation_args["name"] = name - version = get_version(pipfile) # type: Optional[Text] + version = get_version(pipfile) # type: Optional[STRING_TYPE] extras = creation_args.get("extras", None) creation_args["version"] = version req = init_requirement("{0}{1}".format(name, version)) @@ -1119,7 +1113,7 @@ class NamedRequirement(object): @property def line_part(self): - # type: () -> Text + # type: () -> STRING_TYPE # FIXME: This should actually be canonicalized but for now we have to # simply lowercase it and replace underscores, since full canonicalization # also replaces dots and that doesn't actually work when querying the index @@ -1127,7 +1121,7 @@ class NamedRequirement(object): @property def pipfile_part(self): - # type: () -> Dict[Text, Any] + # type: () -> Dict[STRING_TYPE, Any] pipfile_dict = attr.asdict(self, filter=filter_none).copy() # type: ignore if "version" not in pipfile_dict: pipfile_dict["version"] = "*" @@ -1148,36 +1142,36 @@ class FileRequirement(object): containing directories.""" #: Path to the relevant `setup.py` location - setup_path = attr.ib(default=None, cmp=True) # type: Optional[Text] + setup_path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - path = attr.ib(default=None, cmp=True) # type: Optional[Text] + path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: Whether the package is editable editable = attr.ib(default=False, cmp=True) # type: bool #: Extras if applicable - extras = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Tuple[Text] - _uri_scheme = attr.ib(default=None, cmp=True) # type: Optional[Text] + extras = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Tuple[STRING_TYPE, ...] + _uri_scheme = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: URI of the package - uri = attr.ib(cmp=True) # type: Optional[Text] + uri = attr.ib(cmp=True) # type: Optional[STRING_TYPE] #: Link object representing the package to clone link = attr.ib(cmp=True) # type: Optional[Link] #: PyProject Requirements - pyproject_requires = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Tuple + pyproject_requires = attr.ib(factory=tuple, cmp=True) # type: Optional[Tuple[STRING_TYPE, ...]] #: PyProject Build System - pyproject_backend = attr.ib(default=None, cmp=True) # type: Optional[Text] + pyproject_backend = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: PyProject Path - pyproject_path = attr.ib(default=None, cmp=True) # type: Optional[Text] + pyproject_path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: Setup metadata e.g. dependencies _setup_info = attr.ib(default=None, cmp=True) # type: Optional[SetupInfo] _has_hashed_name = attr.ib(default=False, cmp=True) # type: bool _parsed_line = attr.ib(default=None, cmp=False, hash=True) # type: Optional[Line] #: Package name - name = attr.ib(cmp=True) # type: Optional[Text] - #: A :class:`~pkg_resources.Requirement` isntance + name = attr.ib(cmp=True) # type: Optional[STRING_TYPE] + #: A :class:`~pkg_resources.Requirement` instance req = attr.ib(cmp=True) # type: Optional[PackagingRequirement] @classmethod def get_link_from_line(cls, line): - # type: (Text) -> LinkInfo + # type: (STRING_TYPE) -> LinkInfo """Parse link information from given requirement line. Return a 6-tuple: @@ -1211,16 +1205,16 @@ class FileRequirement(object): # Git allows `git@github.com...` lines that are not really URIs. # Add "ssh://" so we can parse correctly, and restore afterwards. - fixed_line = add_ssh_scheme_to_git_uri(line) # type: Text + fixed_line = add_ssh_scheme_to_git_uri(line) # type: STRING_TYPE added_ssh_scheme = fixed_line != line # type: bool # We can assume a lot of things if this is a local filesystem path. if "://" not in fixed_line: p = Path(fixed_line).absolute() # type: Path - path = p.as_posix() # type: Optional[Text] - uri = p.as_uri() # type: Text + path = p.as_posix() # type: Optional[STRING_TYPE] + uri = p.as_uri() # type: STRING_TYPE link = create_link(uri) # type: Link - relpath = None # type: Optional[Text] + relpath = None # type: Optional[STRING_TYPE] try: relpath = get_converted_relative_path(path) except ValueError: @@ -1233,13 +1227,13 @@ class FileRequirement(object): original_url = parsed_url._replace() # type: SplitResult # Split the VCS part out if needed. - original_scheme = parsed_url.scheme # type: Text - vcs_type = None # type: Optional[Text] + original_scheme = parsed_url.scheme # type: STRING_TYPE + vcs_type = None # type: Optional[STRING_TYPE] if "+" in original_scheme: - scheme = None # type: Optional[Text] + scheme = None # type: Optional[STRING_TYPE] vcs_type, _, scheme = original_scheme.partition("+") - parsed_url = parsed_url._replace(scheme=scheme) - prefer = "uri" # type: Text + parsed_url = parsed_url._replace(scheme=scheme) # type: ignore + prefer = "uri" # type: STRING_TYPE else: vcs_type = None prefer = "file" @@ -1261,35 +1255,35 @@ class FileRequirement(object): relpath = None # Cut the fragment, but otherwise this is fixed_line. uri = urllib_parse.urlunsplit( - parsed_url._replace(scheme=original_scheme, fragment="") + parsed_url._replace(scheme=original_scheme, fragment="") # type: ignore ) if added_ssh_scheme: original_uri = urllib_parse.urlunsplit( - original_url._replace(scheme=original_scheme, fragment="") + original_url._replace(scheme=original_scheme, fragment="") # type: ignore ) uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. link = create_link( - urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme)) + urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme)) # type: ignore ) return LinkInfo(vcs_type, prefer, relpath, path, uri, link) @property def setup_py_dir(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self.setup_path: return os.path.dirname(os.path.abspath(self.setup_path)) return None @property def dependencies(self): - # type: () -> Tuple[Dict[Text, PackagingRequirement], List[Union[Text, PackagingRequirement]], List[Text]] - build_deps = [] # type: List[Union[Text, PackagingRequirement]] - setup_deps = [] # type: List[Text] - deps = {} # type: Dict[Text, PackagingRequirement] + # type: () -> Tuple[Dict[S, PackagingRequirement], List[Union[S, PackagingRequirement]], List[S]] + build_deps = [] # type: List[Union[S, PackagingRequirement]] + setup_deps = [] # type: List[S] + deps = {} # type: Dict[S, PackagingRequirement] if self.setup_info: setup_info = self.setup_info.as_dict() deps.update(setup_info.get("requires", {})) @@ -1308,27 +1302,32 @@ class FileRequirement(object): self._setup_info = self.parsed_line.setup_info if self.parsed_line.setup_info.name: self.name = self.parsed_line.setup_info.name - if self.req is None and self._parsed_line.requirement is not None: + if self.req is None and ( + self._parsed_line is not None and self._parsed_line.requirement is not None + ): self.req = self._parsed_line.requirement if self._parsed_line and self._parsed_line.ireq and not self._parsed_line.ireq.req: - if self.req is not None: + if self.req is not None and self._parsed_line._ireq is not None: self._parsed_line._ireq.req = self.req @property def setup_info(self): - # type: () -> SetupInfo + # type: () -> Optional[SetupInfo] from .setup_info import SetupInfo if self._setup_info is None and self.parsed_line: - if self.parsed_line.setup_info: - if not self._parsed_line.setup_info.name: + if self.parsed_line and self._parsed_line and self.parsed_line.setup_info: + if self._parsed_line._setup_info and not self._parsed_line._setup_info.name: self._parsed_line._setup_info.get_info() - self._setup_info = self.parsed_line.setup_info - elif self.parsed_line.ireq and not self.parsed_line.is_wheel: + self._setup_info = self.parsed_line._setup_info + elif self.parsed_line and ( + self.parsed_line.ireq and not self.parsed_line.is_wheel + ): self._setup_info = SetupInfo.from_ireq(self.parsed_line.ireq) else: if self.link and not self.link.is_wheel: self._setup_info = Line(self.line_part).setup_info - self._setup_info.get_info() + if self._setup_info: + self._setup_info.get_info() return self._setup_info @setup_info.setter @@ -1340,7 +1339,7 @@ class FileRequirement(object): @uri.default def get_uri(self): - # type: () -> Text + # type: () -> STRING_TYPE if self.path and not self.uri: self._uri_scheme = "path" return pip_shims.shims.path_to_url(os.path.abspath(self.path)) @@ -1352,13 +1351,16 @@ class FileRequirement(object): @name.default def get_name(self): - # type: () -> Text + # type: () -> STRING_TYPE loc = self.path or self.uri if loc and not self._uri_scheme: self._uri_scheme = "path" if self.path else "file" - name = None - hashed_loc = hashlib.sha256(loc.encode("utf-8")).hexdigest() - hashed_name = hashed_loc[-7:] + name = None # type: Optional[STRING_TYPE] + hashed_loc = None # type: Optional[STRING_TYPE] + hashed_name = None # type: Optional[STRING_TYPE] + if loc: + hashed_loc = hashlib.sha256(loc.encode("utf-8")).hexdigest() + hashed_name = hashed_loc[-7:] if getattr(self, "req", None) and self.req is not None and getattr(self.req, "name") and self.req.name is not None: if self.is_direct_url and self.req.name != hashed_name: return self.req.name @@ -1371,20 +1373,19 @@ class FileRequirement(object): elif self.link and ((self.link.scheme == "file" or self.editable) or ( self.path and self.setup_path and os.path.isfile(str(self.setup_path)) )): - _ireq = None + _ireq = None # type: Optional[InstallRequirement] + target_path = "" # type: STRING_TYPE + if self.setup_py_dir: + target_path = Path(self.setup_py_dir).as_posix() + elif self.path: + target_path = Path(os.path.abspath(self.path)).as_posix() if self.editable: - if self.setup_path: - line = pip_shims.shims.path_to_url(self.setup_py_dir) - else: - line = pip_shims.shims.path_to_url(os.path.abspath(self.path)) + line = pip_shims.shims.path_to_url(target_path) if self.extras: line = "{0}[{1}]".format(line, ",".join(self.extras)) _ireq = pip_shims.shims.install_req_from_editable(line) else: - if self.setup_path: - line = Path(self.setup_py_dir).as_posix() - else: - line = Path(os.path.abspath(self.path)).as_posix() + line = target_path if self.extras: line = "{0}[{1}]".format(line, ",".join(self.extras)) _ireq = pip_shims.shims.install_req_from_line(line) @@ -1400,7 +1401,7 @@ class FileRequirement(object): setupinfo = SetupInfo.from_ireq(_ireq, subdir=subdir) if setupinfo: self._setup_info = setupinfo - self.setup_info.get_info() + self._setup_info.get_info() setupinfo_dict = setupinfo.as_dict() setup_name = setupinfo_dict.get("name", None) if setup_name: @@ -1448,9 +1449,14 @@ class FileRequirement(object): except Exception: pass req = copy.deepcopy(self._parsed_line.requirement) - return req + if req: + return req req = init_requirement(normalize_name(self.name)) + if req is None: + raise ValueError( + "Failed to generate a requirement: missing name for {0!r}".format(self) + ) req.editable = False if self.link is not None: req.line = self.link.url_without_fragment @@ -1519,7 +1525,7 @@ class FileRequirement(object): @property def formatted_path(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self.path: path = self.path if not isinstance(path, Path): @@ -1530,68 +1536,69 @@ class FileRequirement(object): @classmethod def create( cls, - path=None, # type: Optional[Text] - uri=None, # type: Text + path=None, # type: Optional[STRING_TYPE] + uri=None, # type: STRING_TYPE editable=False, # type: bool - extras=None, # type: Optional[Tuple[Text]] + extras=None, # type: Optional[Tuple[STRING_TYPE, ...]] link=None, # type: Link vcs_type=None, # type: Optional[Any] - name=None, # type: Optional[Text] + name=None, # type: Optional[STRING_TYPE] req=None, # type: Optional[Any] - line=None, # type: Optional[Text] - uri_scheme=None, # type: Text + line=None, # type: Optional[STRING_TYPE] + uri_scheme=None, # type: STRING_TYPE setup_path=None, # type: Optional[Any] relpath=None, # type: Optional[Any] parsed_line=None, # type: Optional[Line] ): - # type: (...) -> FileRequirement + # type: (...) -> F if parsed_line is None and line is not None: parsed_line = Line(line) if relpath and not path: path = relpath if not path and uri and link is not None and link.scheme == "file": - path = os.path.abspath(pip_shims.shims.url_to_path(unquote(uri))) + path = os.path.abspath(pip_shims.shims.url_to_path(unquote(uri))) # type: ignore try: path = get_converted_relative_path(path) except ValueError: # Vistir raises a ValueError if it can't make a relpath path = path - if line and not (uri_scheme and uri and link): + if line is not None and not (uri_scheme and uri and link): vcs_type, uri_scheme, relpath, path, uri, link = cls.get_link_from_line(line) if not uri_scheme: uri_scheme = "path" if path else "file" if path and not uri: - uri = unquote(pip_shims.shims.path_to_url(os.path.abspath(path))) - if not link: + uri = unquote(pip_shims.shims.path_to_url(os.path.abspath(path))) # type: ignore + if not link and uri: link = cls.get_link_from_line(uri).link - if not uri: + if not uri and link: uri = unquote(link.url_without_fragment) if not extras: extras = () pyproject_path = None pyproject_requires = None pyproject_backend = None + pyproject_tuple = None # type: Optional[Tuple[STRING_TYPE]] if path is not None: - pyproject_requires = get_pyproject(path) - if pyproject_requires is not None: - pyproject_requires, pyproject_backend = pyproject_requires - pyproject_requires = tuple(pyproject_requires) + pyproject_requires_and_backend = get_pyproject(path) + if pyproject_requires_and_backend is not None: + pyproject_requires, pyproject_backend = pyproject_requires_and_backend if path: setup_paths = get_setup_paths(path) - if setup_paths["pyproject_toml"] is not None: - pyproject_path = Path(setup_paths["pyproject_toml"]) - if setup_paths["setup_py"] is not None: - setup_path = Path(setup_paths["setup_py"]).as_posix() + if isinstance(setup_paths, Mapping): + if "pyproject_toml" in setup_paths and setup_paths["pyproject_toml"]: + pyproject_path = Path(setup_paths["pyproject_toml"]) + if "setup_py" in setup_paths and setup_paths["setup_py"]: + setup_path = Path(setup_paths["setup_py"]).as_posix() if setup_path and isinstance(setup_path, Path): setup_path = setup_path.as_posix() creation_kwargs = { "editable": editable, "extras": extras, "pyproject_path": pyproject_path, - "setup_path": setup_path if setup_path else None, + "setup_path": setup_path, "uri_scheme": uri_scheme, "link": link, "uri": uri, - "pyproject_requires": pyproject_requires, + "pyproject_requires": pyproject_tuple, "pyproject_backend": pyproject_backend, "path": path or relpath, "parsed_line": parsed_line @@ -1600,7 +1607,7 @@ class FileRequirement(object): creation_kwargs["vcs"] = vcs_type if name: creation_kwargs["name"] = name - _line = None # type: Optional[Text] + _line = None # type: Optional[STRING_TYPE] ireq = None # type: Optional[InstallRequirement] setup_info = None # type: Optional[SetupInfo] if parsed_line: @@ -1613,30 +1620,31 @@ class FileRequirement(object): _line = unquote(link.url_without_fragment) if name: _line = "{0}#egg={1}".format(_line, name) - if extras and extras_to_string(extras) not in _line: - _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) - elif uri is not None: + if _line and extras and extras_to_string(extras) not in _line: + _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) # type: ignore + elif isinstance(uri, six.string_types): _line = unquote(uri) - else: + elif line: _line = unquote(line) if editable: - if extras and extras_to_string(extras) not in _line and ( + if _line and extras and extras_to_string(extras) not in _line and ( (link and link.scheme == "file") or (uri and uri.startswith("file")) or (not uri and not link) ): - _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) + _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) # type: ignore if ireq is None: - ireq = pip_shims.shims.install_req_from_editable(_line) + ireq = pip_shims.shims.install_req_from_editable(_line) # type: ignore else: _line = path if (uri_scheme and uri_scheme == "path") else _line - if extras and extras_to_string(extras) not in _line: - _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) + if _line and extras and extras_to_string(extras) not in _line: + _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) # type: ignore if ireq is None: - ireq = pip_shims.shims.install_req_from_line(_line) + ireq = pip_shims.shims.install_req_from_line(_line) # type: ignore if editable: _line = "-e {0}".format(editable) - parsed_line = Line(_line) - if ireq is None: + if _line: + parsed_line = Line(_line) + if ireq is None and parsed_line and parsed_line.ireq: ireq = parsed_line.ireq if extras and ireq is not None and not ireq.extras: ireq.extras = set(extras) @@ -1644,13 +1652,15 @@ class FileRequirement(object): setup_info = SetupInfo.from_ireq(ireq) setupinfo_dict = setup_info.as_dict() setup_name = setupinfo_dict.get("name", None) + build_requires = () # type: Tuple[STRING_TYPE, ...] + build_backend = "" if setup_name is not None: name = setup_name - build_requires = setupinfo_dict.get("build_requires", ()) - build_backend = setupinfo_dict.get("build_backend", "") - if not creation_kwargs.get("pyproject_requires") and build_requires: + build_requires = setupinfo_dict.get("build_requires", build_requires) + build_backend = setupinfo_dict.get("build_backend", build_backend) + if "pyproject_requires" not in creation_kwargs and build_requires: creation_kwargs["pyproject_requires"] = tuple(build_requires) - if not creation_kwargs.get("pyproject_backend") and build_backend: + if "pyproject_backend" not in creation_kwargs and build_backend: creation_kwargs["pyproject_backend"] = build_backend if setup_info is None and parsed_line and parsed_line.setup_info: setup_info = parsed_line.setup_info @@ -1669,12 +1679,11 @@ class FileRequirement(object): name = parsed_line.name if name: creation_kwargs["name"] = name - cls_inst = cls(**creation_kwargs) # type: ignore - return cls_inst + return cls(**creation_kwargs) # type: ignore @classmethod - def from_line(cls, line, extras=None, parsed_line=None): - # type: (Text, Optional[Tuple[Text]], Optional[Line]) -> FileRequirement + def from_line(cls, line, editable=None, extras=None, parsed_line=None): + # type: (AnyStr, Optional[bool], Optional[Tuple[AnyStr, ...]], Optional[Line]) -> F line = line.strip('"').strip("'") link = None path = None @@ -1725,7 +1734,7 @@ class FileRequirement(object): @classmethod def from_pipfile(cls, name, pipfile): - # type: (Text, Dict[Text, Any]) -> FileRequirement + # type: (STRING_TYPE, Dict[STRING_TYPE, Union[Tuple[STRING_TYPE, ...], STRING_TYPE, bool]]) -> F # Parse the values out. After this dance we should have two variables: # path - Local filesystem path. # uri - Absolute URI that is parsable with urlsplit. @@ -1733,7 +1742,7 @@ class FileRequirement(object): uri = pipfile.get("uri") fil = pipfile.get("file") path = pipfile.get("path") - if path: + if path and isinstance(path, six.string_types): if isinstance(path, Path) and not path.is_absolute(): path = get_converted_relative_path(path.as_posix()) elif not os.path.isabs(path): @@ -1757,49 +1766,63 @@ class FileRequirement(object): if not uri: uri = pip_shims.shims.path_to_url(path) - link = cls.get_link_from_line(uri).link + link_info = None # type: Optional[LinkInfo] + if uri and isinstance(uri, six.string_types): + link_info = cls.get_link_from_line(uri) + else: + raise ValueError( + "Failed parsing requirement from pipfile: {0!r}".format(pipfile) + ) + link = None # type: Optional[Link] + if link_info: + link = link_info.link + if link.url_without_fragment: + uri = unquote(link.url_without_fragment) + extras = () # type: Optional[Tuple[STRING_TYPE, ...]] + if "extras" in pipfile: + extras = tuple(pipfile["extras"]) # type: ignore + editable = pipfile["editable"] if "editable" in pipfile else False arg_dict = { "name": name, "path": path, - "uri": unquote(link.url_without_fragment), - "editable": pipfile.get("editable", False), + "uri": uri, + "editable": editable, "link": link, "uri_scheme": uri_scheme, - "extras": pipfile.get("extras", None), + "extras": extras if extras else None, } - extras = pipfile.get("extras", ()) - if extras: - extras = tuple(extras) - line = "" - if pipfile.get("editable", False) and uri_scheme == "path": - line = "{0}".format(path) - if extras: - line = "{0}{1}".format(line, extras_to_string(extras)) + line = "" # type: STRING_TYPE + extras_string = "" if not extras else extras_to_string(extras) + if editable and uri_scheme == "path": + line = "{0}{1}".format(path, extras_string) else: if name: - if extras: - line_name = "{0}{1}".format(name, extras_to_string(extras)) - else: - line_name = "{0}".format(name) + line_name = "{0}{1}".format(name, extras_string) line = "{0}#egg={1}".format(unquote(link.url_without_fragment), line_name) else: - line = unquote(link.url) - if extras: - line = "{0}{1}".format(line, extras_to_string(extras)) + if link: + line = unquote(link.url) + elif uri and isinstance(uri, six.string_types): + line = uri + else: + raise ValueError( + "Failed parsing requirement from pipfile: {0!r}".format(pipfile) + ) + line = "{0}{1}".format(line, extras_string) if "subdirectory" in pipfile: arg_dict["subdirectory"] = pipfile["subdirectory"] line = "{0}&subdirectory={1}".format(line, pipfile["subdirectory"]) - if pipfile.get("editable", False): + if editable: line = "-e {0}".format(line) arg_dict["line"] = line - return cls.create(**arg_dict) + return cls.create(**arg_dict) # type: ignore @property def line_part(self): - # type: () -> Text - link_url = None # type: Optional[Text] - seed = None # type: Optional[Text] + # type: () -> STRING_TYPE + link_url = None # type: Optional[STRING_TYPE] + seed = None # type: Optional[STRING_TYPE] if self.link is not None: link_url = unquote(self.link.url_without_fragment) if self._uri_scheme and self._uri_scheme == "path": @@ -1819,13 +1842,13 @@ class FileRequirement(object): @property def pipfile_part(self): - # type: () -> Dict[Text, Dict[Text, Any]] + # type: () -> Dict[AnyStr, Dict[AnyStr, Any]] excludes = [ "_base_line", "_has_hashed_name", "setup_path", "pyproject_path", "_uri_scheme", "pyproject_requires", "pyproject_backend", "_setup_info", "_parsed_line" ] filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa - pipfile_dict = attr.asdict(self, filter=filter_func).copy() + pipfile_dict = attr.asdict(self, filter=filter_func).copy() # type: Dict name = pipfile_dict.pop("name", None) if name is None: if self.name: @@ -1838,7 +1861,8 @@ class FileRequirement(object): pipfile_dict.pop("_uri_scheme") # For local paths and remote installable artifacts (zipfiles, etc) collision_keys = {"file", "uri", "path"} - collision_order = ["file", "uri", "path"] # type: List[Text] + collision_order = ["file", "uri", "path"] # type: List[STRING_TYPE] + collisions = [] # type: List[STRING_TYPE] key_match = next(iter(k for k in collision_order if k in pipfile_dict.keys())) if self._uri_scheme: dict_key = self._uri_scheme @@ -1880,18 +1904,18 @@ class VCSRequirement(FileRequirement): #: Whether the repository is editable editable = attr.ib(default=None) # type: Optional[bool] #: URI for the repository - uri = attr.ib(default=None) # type: Optional[Text] + uri = attr.ib(default=None) # type: Optional[STRING_TYPE] #: path to the repository, if it's local - path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) # type: Optional[Text] + path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) # type: Optional[STRING_TYPE] #: vcs type, i.e. git/hg/svn - vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) # type: Optional[Text] + vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) # type: Optional[STRING_TYPE] #: vcs reference name (branch / commit / tag) - ref = attr.ib(default=None) # type: Optional[Text] + ref = attr.ib(default=None) # type: Optional[STRING_TYPE] #: Subdirectory to use for installation if applicable - subdirectory = attr.ib(default=None) # type: Optional[Text] + subdirectory = attr.ib(default=None) # type: Optional[STRING_TYPE] _repo = attr.ib(default=None) # type: Optional[VCSRepository] - _base_line = attr.ib(default=None) # type: Optional[Text] - name = attr.ib() # type: Text + _base_line = attr.ib(default=None) # type: Optional[STRING_TYPE] + name = attr.ib() # type: STRING_TYPE link = attr.ib() # type: Optional[pip_shims.shims.Link] req = attr.ib() # type: Optional[RequirementType] @@ -1911,6 +1935,15 @@ class VCSRequirement(FileRequirement): new_uri = "{0}{1}".format(vcs_type, new_uri) self.uri = new_uri + @property + def url(self): + # type: () -> STRING_TYPE + if self.link and self.link.url: + return self.link.url + elif self.uri: + return self.uri + raise ValueError("No valid url found for requirement {0!r}".format(self)) + @link.default def get_link(self): # type: () -> pip_shims.shims.Link @@ -1927,19 +1960,20 @@ class VCSRequirement(FileRequirement): @name.default def get_name(self): - # type: () -> Optional[Text] - return ( - self.link.egg_fragment or self.req.name - if getattr(self, "req", None) - else super(VCSRequirement, self).get_name() - ) + # type: () -> STRING_TYPE + if self.link and self.link.egg_fragment: + return self.link.egg_fragment + if self.req and self.req.name: + return self.req.name + return super(VCSRequirement, self).get_name() @property def vcs_uri(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] uri = self.uri - if not any(uri.startswith("{0}+".format(vcs)) for vcs in VCS_LIST): - uri = "{0}+{1}".format(self.vcs, uri) + if uri and not any(uri.startswith("{0}+".format(vcs)) for vcs in VCS_LIST): + if self.vcs: + uri = "{0}+{1}".format(self.vcs, uri) return uri @property @@ -1967,7 +2001,11 @@ class VCSRequirement(FileRequirement): @req.default def get_requirement(self): # type: () -> PackagingRequirement - name = self.name or self.link.egg_fragment + name = None # type: Optional[STRING_TYPE] + if self.name: + name = self.name + elif self.link and self.link.egg_fragment: + name = self.link.egg_fragment url = None if self.uri: url = self.uri @@ -1985,8 +2023,10 @@ class VCSRequirement(FileRequirement): if url is not None: url = add_ssh_scheme_to_git_uri(url) elif self.uri is not None: - url = self.parse_link_from_line(self.uri).link.url_without_fragment - if url.startswith("git+file:/") and not url.startswith("git+file:///"): + link = self.get_link_from_line(self.uri).link + if link: + url = link.url_without_fragment + if url and url.startswith("git+file:/") and not url.startswith("git+file:///"): url = url.replace("git+file:/", "git+file:///") if url: req.url = url @@ -2004,13 +2044,14 @@ class VCSRequirement(FileRequirement): req.path = self.path req.link = self.link if ( + self.link and self.link.url_without_fragment and self.uri and self.uri != unquote(self.link.url_without_fragment) and "git+ssh://" in self.link.url and "git+git@" in self.uri ): req.line = self.uri url = self.link.url_without_fragment - if url.startswith("git+file:/") and not url.startswith("git+file:///"): + if url and url.startswith("git+file:/") and not url.startswith("git+file:///"): url = url.replace("git+file:/", "git+file:///") req.url = url return req @@ -2028,7 +2069,7 @@ class VCSRequirement(FileRequirement): return self._repo def get_checkout_dir(self, src_dir=None): - # type: (Optional[Text]) -> Text + # type: (Optional[S]) -> STRING_TYPE src_dir = os.environ.get("PIP_SRC", None) if not src_dir else src_dir checkout_dir = None if self.is_local: @@ -2037,21 +2078,25 @@ class VCSRequirement(FileRequirement): path = pip_shims.shims.url_to_path(self.uri) if path and os.path.exists(path): checkout_dir = os.path.abspath(path) - return checkout_dir + return vistir.compat.fs_encode(checkout_dir) if src_dir is not None: - checkout_dir = os.path.join(os.path.abspath(src_dir), self.name) - mkdir_p(src_dir) + checkout_dir = os.path.join( + os.path.abspath(src_dir), self.name + ) + mkdir_p(vistir.compat.fs_encode(src_dir)) return checkout_dir - return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) + return vistir.compat.fs_encode( + os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) + ) def get_vcs_repo(self, src_dir=None, checkout_dir=None): - # type: (Optional[Text], Optional[Text]) -> VCSRepository + # type: (Optional[STRING_TYPE], STRING_TYPE) -> VCSRepository from .vcs import VCSRepository if checkout_dir is None: checkout_dir = self.get_checkout_dir(src_dir=src_dir) vcsrepo = VCSRepository( - url=self.link.url, + url=self.url, name=self.name, ref=self.ref if self.ref else None, checkout_directory=checkout_dir, @@ -2076,13 +2121,13 @@ class VCSRequirement(FileRequirement): return vcsrepo def get_commit_hash(self): - # type: () -> Text + # type: () -> STRING_TYPE hash_ = None hash_ = self.repo.get_commit_hash() return hash_ def update_repo(self, src_dir=None, ref=None): - # type: (Optional[Text], Optional[Text]) -> Text + # type: (Optional[STRING_TYPE], Optional[STRING_TYPE]) -> STRING_TYPE if ref: self.ref = ref else: @@ -2092,12 +2137,13 @@ class VCSRequirement(FileRequirement): if not self.is_local and ref is not None: self.repo.checkout_ref(ref) repo_hash = self.repo.get_commit_hash() - self.req.revision = repo_hash + if self.req: + self.req.revision = repo_hash return repo_hash @contextmanager def locked_vcs_repo(self, src_dir=None): - # type: (Optional[Text]) -> Generator[VCSRepository, None, None] + # type: (Optional[AnyStr]) -> Generator[VCSRepository, None, None] if not src_dir: src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") vcsrepo = self.get_vcs_repo(src_dir=src_dir) @@ -2109,7 +2155,7 @@ class VCSRequirement(FileRequirement): revision = self.req.revision = vcsrepo.get_commit_hash() # Remove potential ref in the end of uri after ref is parsed - if "@" in self.link.show_url and "@" in self.uri: + if self.link and "@" in self.link.show_url and self.uri and "@" in self.uri: uri, ref = split_ref_from_uri(self.uri) checkout = revision if checkout and ref and ref in checkout: @@ -2119,15 +2165,12 @@ class VCSRequirement(FileRequirement): if self._parsed_line: self._parsed_line.vcsrepo = vcsrepo if self._setup_info: - _old_setup_info = self._setup_info self._setup_info = attr.evolve( self._setup_info, requirements=(), _extras_requirements=(), build_requires=(), setup_requires=(), version=None, metadata=None ) - if self.parsed_line: + if self.parsed_line and self._parsed_line: self._parsed_line.vcsrepo = vcsrepo - # self._parsed_line._specifier = "=={0}".format(self.setup_info.version) - # self._parsed_line.specifiers = self._parsed_line._specifier if self.req: self.req.specifier = SpecifierSet("=={0}".format(self.setup_info.version)) try: @@ -2138,8 +2181,8 @@ class VCSRequirement(FileRequirement): @classmethod def from_pipfile(cls, name, pipfile): - # type: (Text, Dict[Text, Union[List[Text], Text, bool]]) -> VCSRequirement - creation_args = {} + # type: (STRING_TYPE, Dict[S, Union[Tuple[S, ...], S, bool]]) -> F + creation_args = {} # type: Dict[STRING_TYPE, CREATION_ARG_TYPES] pipfile_keys = [ k for k in ( @@ -2155,38 +2198,42 @@ class VCSRequirement(FileRequirement): + VCS_LIST if k in pipfile ] + # extras = None # type: Optional[Tuple[STRING_TYPE, ...]] for key in pipfile_keys: - if key == "extras": - extras = pipfile.get(key, None) - if extras: - pipfile[key] = sorted(dedup([extra.lower() for extra in extras])) - if key in VCS_LIST: - creation_args["vcs"] = key - target = pipfile.get(key) - drive, path = os.path.splitdrive(target) - if ( - not drive - and not os.path.exists(target) - and ( - is_valid_url(target) - or is_file_url(target) - or target.startswith("git@") - ) - ): - creation_args["uri"] = target + if key == "extras" and key in pipfile: + extras = pipfile[key] + if isinstance(extras, (list, tuple)): + pipfile[key] = tuple(sorted({extra.lower() for extra in extras})) else: - creation_args["path"] = target - if os.path.isabs(target): - creation_args["uri"] = pip_shims.shims.path_to_url(target) - else: - creation_args[key] = pipfile.get(key) + pipfile[key] = extras + if key in VCS_LIST and key in pipfile_keys: + creation_args["vcs"] = key + target = pipfile[key] + if isinstance(target, six.string_types): + drive, path = os.path.splitdrive(target) + if ( + not drive + and not os.path.exists(target) + and ( + is_valid_url(target) + or is_file_url(target) + or target.startswith("git@") + ) + ): + creation_args["uri"] = target + else: + creation_args["path"] = target + if os.path.isabs(target): + creation_args["uri"] = pip_shims.shims.path_to_url(target) + elif key in pipfile_keys: + creation_args[key] = pipfile[key] creation_args["name"] = name - cls_inst = cls(**creation_args) + cls_inst = cls(**creation_args) # type: ignore return cls_inst @classmethod def from_line(cls, line, editable=None, extras=None, parsed_line=None): - # type: (Text, Optional[bool], Optional[Tuple[Text]], Optional[Line]) -> VCSRequirement + # type: (AnyStr, Optional[bool], Optional[Tuple[AnyStr, ...]], Optional[Line]) -> F relpath = None if parsed_line is None: parsed_line = Line(line) @@ -2211,12 +2258,15 @@ class VCSRequirement(FileRequirement): name, extras = pip_shims.shims._strip_extras(link.egg_fragment) else: name, _ = pip_shims.shims._strip_extras(link.egg_fragment) - if extras: - extras = parse_extras(extras) - else: + parsed_extras = None # type: Optional[List[STRING_TYPE]] + extras_tuple = None # type: Optional[Tuple[STRING_TYPE, ...]] + if not extras: line, extras = pip_shims.shims._strip_extras(line) if extras: - extras = tuple(extras) + if isinstance(extras, six.string_types): + parsed_extras = parse_extras(extras) + if parsed_extras: + extras_tuple = tuple(parsed_extras) subdirectory = link.subdirectory_fragment ref = None if uri: @@ -2232,7 +2282,7 @@ class VCSRequirement(FileRequirement): "name": name if name else parsed_line.name, "path": relpath or path, "editable": editable, - "extras": extras, + "extras": extras_tuple, "link": link, "vcs_type": vcs_type, "line": line, @@ -2252,7 +2302,7 @@ class VCSRequirement(FileRequirement): path=relpath or path, editable=editable, uri=uri, - extras=extras, + extras=extras_tuple if extras_tuple else tuple(), base_line=line, parsed_line=parsed_line ) @@ -2264,21 +2314,25 @@ class VCSRequirement(FileRequirement): @property def line_part(self): - # type: () -> Text + # type: () -> STRING_TYPE """requirements.txt compatible line part sans-extras""" + base = "" # type: STRING_TYPE if self.is_local: base_link = self.link if not self.link: base_link = self.get_link() - final_format = ( - "{{0}}#egg={0}".format(base_link.egg_fragment) - if base_link.egg_fragment - else "{0}" - ) + if base_link and base_link.egg_fragment: + final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) + else: + final_format = "{0}" base = final_format.format(self.vcs_uri) - elif self._parsed_line is not None and self._parsed_line.is_direct_url: + elif self._parsed_line is not None and ( + self._parsed_line.is_direct_url and self._parsed_line.line_with_prefix + ): return self._parsed_line.line_with_prefix - elif getattr(self, "_base_line", None): + elif getattr(self, "_base_line", None) and ( + isinstance(self._base_line, six.string_types) + ): base = self._base_line else: base = getattr(self, "link", self.get_link()).url @@ -2295,20 +2349,28 @@ class VCSRequirement(FileRequirement): @staticmethod def _choose_vcs_source(pipfile): - # type: (Dict[Text, Union[List[Text], Text, bool]]) -> Dict[Text, Union[List[Text], Text, bool]] + # type: (Dict[S, Union[S, Any]]) -> Dict[S, Union[S, Any]] src_keys = [k for k in pipfile.keys() if k in ["path", "uri", "file"]] + vcs_type = "" # type: Optional[STRING_TYPE] + alt_type = "" # type: Optional[STRING_TYPE] + vcs_value = "" # type: STRING_TYPE if src_keys: chosen_key = first(src_keys) vcs_type = pipfile.pop("vcs") - _, pipfile_url = split_vcs_method_from_uri(pipfile.get(chosen_key)) - pipfile[vcs_type] = pipfile_url + if chosen_key in pipfile: + vcs_value = pipfile[chosen_key] + alt_type, pipfile_url = split_vcs_method_from_uri(vcs_value) + if vcs_type is None: + vcs_type = alt_type + if vcs_type and pipfile_url: + pipfile[vcs_type] = pipfile_url for removed in src_keys: pipfile.pop(removed) return pipfile @property def pipfile_part(self): - # type: () -> Dict[Text, Dict[Text, Union[List[Text], Text, bool]]] + # type: () -> Dict[S, Dict[S, Union[List[S], S, bool, RequirementType, pip_shims.shims.Link]]] excludes = [ "_repo", "_base_line", "setup_path", "_has_hashed_name", "pyproject_path", "pyproject_requires", "pyproject_backend", "_setup_info", "_parsed_line", @@ -2327,20 +2389,20 @@ class VCSRequirement(FileRequirement): if "vcs" in pipfile_dict: pipfile_dict = self._choose_vcs_source(pipfile_dict) name, _ = pip_shims.shims._strip_extras(name) - return {name: pipfile_dict} + return {name: pipfile_dict} # type: ignore @attr.s(cmp=True, hash=True) class Requirement(object): - _name = attr.ib(cmp=True) # type: Text - vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs), cmp=True) # type: Optional[Text] + _name = attr.ib(cmp=True) # type: STRING_TYPE + vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs), cmp=True) # type: Optional[STRING_TYPE] req = attr.ib(default=None, cmp=True) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] - markers = attr.ib(default=None, cmp=True) # type: Optional[Text] - _specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers), cmp=True) # type: Optional[Text] - index = attr.ib(default=None, cmp=True) # type: Optional[Text] + markers = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] + _specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers), cmp=True) # type: Optional[STRING_TYPE] + index = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] editable = attr.ib(default=None, cmp=True) # type: Optional[bool] - hashes = attr.ib(factory=frozenset, converter=frozenset, cmp=True) # type: Optional[Tuple[Text]] - extras = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Optional[Tuple[Text]] + hashes = attr.ib(factory=frozenset, converter=frozenset, cmp=True) # type: FrozenSet[STRING_TYPE] + extras = attr.ib(factory=tuple, cmp=True) # type: Tuple[STRING_TYPE, ...] abstract_dep = attr.ib(default=None, cmp=False) # type: Optional[AbstractDependency] _line_instance = attr.ib(default=None, cmp=False) # type: Optional[Line] _ireq = attr.ib(default=None, cmp=False) # type: Optional[pip_shims.InstallRequirement] @@ -2350,12 +2412,14 @@ class Requirement(object): @_name.default def get_name(self): - # type: () -> Optional[Text] - return self.req.name + # type: () -> Optional[STRING_TYPE] + if self.req is not None: + return self.req.name + return None @property def name(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self._name is not None: return self._name name = None @@ -2369,32 +2433,44 @@ class Requirement(object): @property def requirement(self): # type: () -> Optional[PackagingRequirement] - return self.req.req + if self.req: + return self.req.req + return None def add_hashes(self, hashes): - # type: (Union[List, Set, Tuple]) -> Requirement + # type: (Union[S, List[S], Set[S], Tuple[S, ...]]) -> Requirement + new_hashes = set() # type: Set[STRING_TYPE] + if self.hashes is not None: + new_hashes |= set(self.hashes) if isinstance(hashes, six.string_types): - new_hashes = set(self.hashes).add(hashes) + new_hashes.add(hashes) else: - new_hashes = set(self.hashes) | set(hashes) - return attr.evolve(self, hashes=frozenset(new_hashes)) + new_hashes |= set(hashes) + return attr.evolve(self, hashes=tuple(new_hashes)) def get_hashes_as_pip(self, as_list=False): - # type: (bool) -> Union[Text, List[Text]] - if self.hashes: - if as_list: - return [HASH_STRING.format(h) for h in self.hashes] - return "".join([HASH_STRING.format(h) for h in self.hashes]) - return "" if not as_list else [] + # type: (bool) -> Union[STRING_TYPE, List[STRING_TYPE]] + hashes = "" # type: Union[STRING_TYPE, List[STRING_TYPE]] + if as_list: + hashes = [] + if self.hashes: + hashes = [HASH_STRING.format(h) for h in self.hashes] + else: + hashes = "" + if self.hashes: + hashes = "".join([HASH_STRING.format(h) for h in self.hashes]) + return hashes @property def hashes_as_pip(self): - # type: () -> Union[Text, List[Text]] - self.get_hashes_as_pip() + # type: () -> STRING_TYPE + hashes = self.get_hashes_as_pip() + assert isinstance(hashes, six.string_types) + return hashes @property def markers_as_pip(self): - # type: () -> Text + # type: () -> S if self.markers: return " ; {0}".format(self.markers).replace('"', "'") @@ -2402,27 +2478,28 @@ class Requirement(object): @property def extras_as_pip(self): - # type: () -> Text + # type: () -> STRING_TYPE if self.extras: return "[{0}]".format( - ",".join(sorted([extra.lower() for extra in self.extras])) + ",".join(sorted([extra.lower() for extra in self.extras])) # type: ignore ) return "" @cached_property def commit_hash(self): - # type: () -> Optional[Text] - if not self.is_vcs: + # type: () -> Optional[S] + if self.req is None or not isinstance(self.req, VCSRequirement): return None commit_hash = None - with self.req.locked_vcs_repo() as repo: - commit_hash = repo.get_commit_hash() + if self.req is not None: + with self.req.locked_vcs_repo() as repo: + commit_hash = repo.get_commit_hash() return commit_hash @_specifiers.default def get_specifiers(self): - # type: () -> Text + # type: () -> S if self.req and self.req.req and self.req.req.specifier: return specs_to_string(self.req.req.specifier) return "" @@ -2449,8 +2526,8 @@ class Requirement(object): def line_instance(self): # type: () -> Optional[Line] if self._line_instance is None: - if self.req._parsed_line is not None: - self._line_instance = self.req.parsed_line + if self.req is not None and self.req._parsed_line is not None: + self._line_instance = self.req._parsed_line else: include_extras = True include_specifiers = True @@ -2458,16 +2535,17 @@ class Requirement(object): include_extras = False if self.is_file_or_url or self.is_vcs or not self._specifiers: include_specifiers = False - + line_part = "" # type: STRING_TYPE + if self.req and self.req.line_part: + line_part = "{0!s}".format(self.req.line_part) + parts = [] # type: List[STRING_TYPE] parts = [ - self.req.line_part, + line_part, self.extras_as_pip if include_extras else "", - self._specifiers if include_specifiers else "", + self._specifiers if include_specifiers and self._specifiers else "", self.markers_as_pip, ] line = "".join(parts) - if line is None: - return None self._line_instance = Line(line) return self._line_instance @@ -2480,7 +2558,7 @@ class Requirement(object): @property def specifiers(self): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] if self._specifiers: return self._specifiers else: @@ -2488,28 +2566,38 @@ class Requirement(object): if specs: self._specifiers = specs return specs - if self.is_named and not self._specifiers: + if not self._specifiers and ( + self.req is not None and + isinstance(self.req, NamedRequirement) and + self.req.version + ): self._specifiers = self.req.version - elif not self.editable and not self.is_named: + elif not self.editable and self.req and ( + not isinstance(self.req, NamedRequirement) and self.req.setup_info + ): if self.line_instance and self.line_instance.setup_info and self.line_instance.setup_info.version: self._specifiers = "=={0}".format(self.req.setup_info.version) - elif self.req.parsed_line.specifiers and not self._specifiers: - self._specifiers = specs_to_string(self.req.parsed_line.specifiers) - elif self.line_instance.specifiers and not self._specifiers: - self._specifiers = specs_to_string(self.line_instance.specifiers) - elif not self._specifiers and (self.is_file_or_url or self.is_vcs): - try: - setupinfo_dict = self.run_requires() - except Exception: - setupinfo_dict = None - if setupinfo_dict is not None: - self._specifiers = "=={0}".format(setupinfo_dict.get("version")) + elif not self._specifiers: + if self.req and self.req.parsed_line and self.req.parsed_line.specifiers: + self._specifiers = specs_to_string(self.req.parsed_line.specifiers) + elif self.line_instance and self.line_instance.specifiers: + self._specifiers = specs_to_string(self.line_instance.specifiers) + elif self.is_file_or_url or self.is_vcs: + try: + setupinfo_dict = self.run_requires() + except Exception: + setupinfo_dict = None + if setupinfo_dict is not None: + self._specifiers = "=={0}".format(setupinfo_dict.get("version")) if self._specifiers: specset = SpecifierSet(self._specifiers) if self.line_instance and not self.line_instance.specifiers: self.line_instance.specifiers = specset - if self.req and self.req.parsed_line and not self.req.parsed_line.specifiers: - self.req._parsed_line.specifiers = specset + if self.req: + if self.req._parsed_line and not self.req._parsed_line.specifiers: + self.req._parsed_line.specifiers = specset + elif not self.req._parsed_line and self.line_instance: + self.req._parsed_line = self.line_instance if self.req and self.req.req and not self.req.req.specifier: self.req.req.specifier = specset return self._specifiers @@ -2521,9 +2609,10 @@ class Requirement(object): @property def build_backend(self): - # type: () -> Optional[Text] - if self.is_vcs or (self.is_file_or_url and ( - self.req is not None and self.req.is_local)): + # type: () -> Optional[STRING_TYPE] + if self.req is not None and ( + not isinstance(self.req, NamedRequirement) and self.req.is_local + ): setup_info = self.run_requires() build_backend = setup_info.get("build_backend") return build_backend @@ -2549,8 +2638,7 @@ class Requirement(object): @property def is_wheel(self): # type: () -> bool - if not self.is_named and ( - self.req is not None and + if self.req and not isinstance(self.req, NamedRequirement) and ( self.req.link is not None and self.req.link.is_wheel ): @@ -2559,7 +2647,7 @@ class Requirement(object): @property def normalized_name(self): - # type: () -> Text + # type: () -> S return canonicalize_name(self.name) def copy(self): @@ -2568,7 +2656,7 @@ class Requirement(object): @classmethod @lru_cache() def from_line(cls, line): - # type: (Text) -> Requirement + # type: (AnyStr) -> Requirement if isinstance(line, pip_shims.shims.InstallRequirement): line = format_requirement(line) parsed_line = Line(line) @@ -2588,6 +2676,7 @@ class Requirement(object): req_markers = PackagingRequirement("fakepkg; {0}".format(parsed_line.markers)) if r is not None and r.req is not None: r.req.marker = getattr(req_markers, "marker", None) if req_markers else None + args = {} # type: Dict[STRING_TYPE, CREATION_ARG_TYPES] args = { "name": r.name, "vcs": parsed_line.vcs, @@ -2597,6 +2686,7 @@ class Requirement(object): "line_instance": parsed_line } if parsed_line.extras: + extras = () # type: Tuple[STRING_TYPE, ...] extras = tuple(sorted(dedup([extra.lower() for extra in parsed_line.extras]))) args["extras"] = extras if r is not None: @@ -2896,12 +2986,14 @@ class Requirement(object): ) def find_all_matches(self, sources=None, finder=None): + # type: (Optional[List[Dict[S, Union[S, bool]]]], Optional[PackageFinder]) -> List[InstallationCandidate] """Find all matching candidates for the current requirement. Consults a finder to find all matching candidates. :param sources: Pipfile-formatted sources, defaults to None :param sources: list[dict], optional + :param PackageFinder finder: A **PackageFinder** instance from pip's repository implementation :return: A list of Installation Candidates :rtype: list[ :class:`~pip._internal.index.InstallationCandidate` ] """ @@ -2935,36 +3027,43 @@ class Requirement(object): return info_dict def merge_markers(self, markers): + # type: (Union[AnyStr, Marker]) -> None if not isinstance(markers, Marker): markers = Marker(markers) - _markers = set(Marker(self.ireq.markers)) if self.ireq.markers else set(markers) + _markers = set() # type: Set[Marker] + if self.ireq and self.ireq.markers: + _markers.add(Marker(self.ireq.markers)) _markers.add(markers) new_markers = Marker(" or ".join([str(m) for m in sorted(_markers)])) self.markers = str(new_markers) - self.req.req.marker = new_markers + if self.req and self.req.req: + self.req.req.marker = new_markers + return def file_req_from_parsed_line(parsed_line): # type: (Line) -> FileRequirement path = parsed_line.relpath if parsed_line.relpath else parsed_line.path - pyproject_requires = () # type: Tuple[Text] + pyproject_requires = None # type: Optional[Tuple[STRING_TYPE, ...]] if parsed_line.pyproject_requires is not None: pyproject_requires = tuple(parsed_line.pyproject_requires) - return FileRequirement( - setup_path=parsed_line.setup_py, - path=path, - editable=parsed_line.editable, - extras=parsed_line.extras, - uri_scheme=parsed_line.preferred_scheme, - link=parsed_line.link, - uri=parsed_line.uri, - pyproject_requires=pyproject_requires, - pyproject_backend=parsed_line.pyproject_backend, - pyproject_path=Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None, - parsed_line=parsed_line, - name=parsed_line.name, - req=parsed_line.requirement - ) + req_dict = { + "setup_path": parsed_line.setup_py, + "path": path, + "editable": parsed_line.editable, + "extras": parsed_line.extras, + "uri_scheme": parsed_line.preferred_scheme, + "link": parsed_line.link, + "uri": parsed_line.uri, + "pyproject_requires": pyproject_requires, + "pyproject_backend": parsed_line.pyproject_backend, + "pyproject_path": Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None, + "parsed_line": parsed_line, + "req": parsed_line.requirement + } + if parsed_line.name is not None: + req_dict["name"] = parsed_line.name + return FileRequirement(**req_dict) # type: ignore def vcs_req_from_parsed_line(parsed_line): @@ -2983,37 +3082,41 @@ def vcs_req_from_parsed_line(parsed_line): )) else: link = parsed_line.link - pyproject_requires = () # type: Tuple[Text] + pyproject_requires = () # type: Optional[Tuple[STRING_TYPE, ...]] if parsed_line.pyproject_requires is not None: pyproject_requires = tuple(parsed_line.pyproject_requires) - return VCSRequirement( - setup_path=parsed_line.setup_py, - path=parsed_line.path, - editable=parsed_line.editable, - vcs=parsed_line.vcs, - ref=parsed_line.ref, - subdirectory=parsed_line.subdirectory, - extras=parsed_line.extras, - uri_scheme=parsed_line.preferred_scheme, - link=link, - uri=parsed_line.uri, - pyproject_requires=pyproject_requires, - pyproject_backend=parsed_line.pyproject_backend, - pyproject_path=Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None, - parsed_line=parsed_line, - name=parsed_line.name, - req=parsed_line.requirement, - base_line=line, - ) + vcs_dict = { + "setup_path": parsed_line.setup_py, + "path": parsed_line.path, + "editable": parsed_line.editable, + "vcs": parsed_line.vcs, + "ref": parsed_line.ref, + "subdirectory": parsed_line.subdirectory, + "extras": parsed_line.extras, + "uri_scheme": parsed_line.preferred_scheme, + "link": link, + "uri": parsed_line.uri, + "pyproject_requires": pyproject_requires, + "pyproject_backend": parsed_line.pyproject_backend, + "pyproject_path": Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None, + "parsed_line": parsed_line, + "req": parsed_line.requirement, + "base_line": line + } + if parsed_line.name: + vcs_dict["name"] = parsed_line.name + return VCSRequirement(**vcs_dict) # type: ignore def named_req_from_parsed_line(parsed_line): # type: (Line) -> NamedRequirement - return NamedRequirement( - name=parsed_line.name, - version=parsed_line.specifier, - req=parsed_line.requirement, - extras=parsed_line.extras, - editable=parsed_line.editable, - parsed_line=parsed_line - ) + if parsed_line.name is not None: + return NamedRequirement( + name=parsed_line.name, + version=parsed_line.specifier, + req=parsed_line.requirement, + extras=parsed_line.extras, + editable=parsed_line.editable, + parsed_line=parsed_line + ) + return NamedRequirement.from_line(parsed_line.line) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index e599f6f7..f278afd2 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -49,7 +49,7 @@ except ImportError: if MYPY_RUNNING: - from typing import Any, Dict, List, Generator, Optional, Union, Tuple, TypeVar, Text, Set + from typing import Any, Dict, List, Generator, Optional, Union, Tuple, TypeVar, Text, Set, AnyStr from pip_shims.shims import InstallRequirement, PackageFinder from pkg_resources import ( PathMetadata, DistInfoDistribution, Requirement as PkgResourcesRequirement @@ -58,6 +58,8 @@ if MYPY_RUNNING: TRequirement = TypeVar("TRequirement") RequirementType = TypeVar('RequirementType', covariant=True, bound=PackagingRequirement) MarkerType = TypeVar('MarkerType', covariant=True, bound=Marker) + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) @@ -69,7 +71,7 @@ _setup_distribution = None def pep517_subprocess_runner(cmd, cwd=None, extra_environ=None): - # type: (List[Text], Optional[Text], Optional[Dict[Text, Text]]) -> None + # type: (List[AnyStr], Optional[AnyStr], Optional[Dict[AnyStr, AnyStr]]) -> None """The default method of calling the wrapper subprocess.""" env = os.environ.copy() if extra_environ: @@ -135,7 +137,7 @@ def build_pep517(source_dir, build_dir, config_settings=None, dist_type="wheel") @ensure_mkdir_p(mode=0o775) def _get_src_dir(root): - # type: (Text) -> Text + # type: (AnyStr) -> AnyStr src = os.environ.get("PIP_SRC") if src: return src @@ -152,7 +154,7 @@ def _get_src_dir(root): @lru_cache() def ensure_reqs(reqs): - # type: (List[Union[Text, PkgResourcesRequirement]]) -> List[PkgResourcesRequirement] + # type: (List[Union[S, PkgResourcesRequirement]]) -> List[PkgResourcesRequirement] import pkg_resources if not isinstance(reqs, Iterable): raise TypeError("Expecting an Iterable, got %r" % reqs) @@ -168,18 +170,18 @@ def ensure_reqs(reqs): def _prepare_wheel_building_kwargs(ireq=None, src_root=None, src_dir=None, editable=False): - # type: (Optional[InstallRequirement], Optional[Text], Optional[Text], bool) -> Dict[Text, Text] - download_dir = os.path.join(CACHE_DIR, "pkgs") # type: Text + # type: (Optional[InstallRequirement], Optional[AnyStr], Optional[AnyStr], bool) -> Dict[AnyStr, AnyStr] + download_dir = os.path.join(CACHE_DIR, "pkgs") # type: STRING_TYPE mkdir_p(download_dir) - wheel_download_dir = os.path.join(CACHE_DIR, "wheels") # type: Text + wheel_download_dir = os.path.join(CACHE_DIR, "wheels") # type: STRING_TYPE mkdir_p(wheel_download_dir) if src_dir is None: if editable and src_root is not None: src_dir = src_root elif ireq is None and src_root is not None: - src_dir = _get_src_dir(root=src_root) # type: Text + src_dir = _get_src_dir(root=src_root) # type: STRING_TYPE elif ireq is not None and ireq.editable and src_root is not None: src_dir = _get_src_dir(root=src_root) else: @@ -199,7 +201,7 @@ def _prepare_wheel_building_kwargs(ireq=None, src_root=None, src_dir=None, edita def iter_metadata(path, pkg_name=None, metadata_type="egg-info"): - # type: (Text, Optional[Text], Text) -> Generator + # type: (AnyStr, Optional[AnyStr], AnyStr) -> Generator if pkg_name is not None: pkg_variants = get_name_variants(pkg_name) non_matching_dirs = [] @@ -217,7 +219,7 @@ def iter_metadata(path, pkg_name=None, metadata_type="egg-info"): def find_egginfo(target, pkg_name=None): - # type: (Text, Optional[Text]) -> Generator + # type: (AnyStr, Optional[AnyStr]) -> Generator egg_dirs = ( egg_dir for egg_dir in iter_metadata(target, pkg_name=pkg_name) if egg_dir is not None @@ -230,7 +232,7 @@ def find_egginfo(target, pkg_name=None): def find_distinfo(target, pkg_name=None): - # type: (Text, Optional[Text]) -> Generator + # type: (AnyStr, Optional[AnyStr]) -> Generator dist_dirs = ( dist_dir for dist_dir in iter_metadata(target, pkg_name=pkg_name, metadata_type="dist-info") if dist_dir is not None @@ -243,7 +245,7 @@ def find_distinfo(target, pkg_name=None): def get_metadata(path, pkg_name=None, metadata_type=None): - # type: (Text, Optional[Text], Optional[Text]) -> Dict[Text, Union[Text, List[RequirementType], Dict[Text, RequirementType]]] + # type: (S, Optional[S], Optional[S]) -> Dict[S, Union[S, List[RequirementType], Dict[S, RequirementType]]] metadata_dirs = [] wheel_allowed = metadata_type == "wheel" or metadata_type is None egg_allowed = metadata_type == "egg" or metadata_type is None @@ -279,7 +281,7 @@ def get_metadata(path, pkg_name=None, metadata_type=None): @lru_cache() def get_extra_name_from_marker(marker): - # type: (MarkerType) -> Optional[Text] + # type: (MarkerType) -> Optional[S] if not marker: raise ValueError("Invalid value for marker: {0!r}".format(marker)) if not getattr(marker, "_markers", None): @@ -291,7 +293,7 @@ def get_extra_name_from_marker(marker): def get_metadata_from_wheel(wheel_path): - # type: (Text) -> Dict[Any, Any] + # type: (S) -> Dict[Any, Any] if not isinstance(wheel_path, six.string_types): raise TypeError("Expected string instance, received {0!r}".format(wheel_path)) try: @@ -327,7 +329,7 @@ def get_metadata_from_wheel(wheel_path): def get_metadata_from_dist(dist): - # type: (Union[PathMetadata, DistInfoDistribution]) -> Dict[Text, Union[Text, List[RequirementType], Dict[Text, RequirementType]]] + # type: (Union[PathMetadata, DistInfoDistribution]) -> Dict[S, Union[S, List[RequirementType], Dict[S, RequirementType]]] try: requires = dist.requires() except Exception: @@ -366,25 +368,25 @@ def get_metadata_from_dist(dist): @attr.s(slots=True, frozen=True) class BaseRequirement(object): - name = attr.ib(default="", cmp=True) # type: Text + name = attr.ib(default="", cmp=True) # type: STRING_TYPE requirement = attr.ib(default=None, cmp=True) # type: Optional[PkgResourcesRequirement] def __str__(self): - # type: () -> Text + # type: () -> S return "{0}".format(str(self.requirement)) def as_dict(self): - # type: () -> Dict[Text, Optional[PkgResourcesRequirement]] + # type: () -> Dict[S, Optional[PkgResourcesRequirement]] return {self.name: self.requirement} def as_tuple(self): - # type: () -> Tuple[Text, Optional[PkgResourcesRequirement]] + # type: () -> Tuple[S, Optional[PkgResourcesRequirement]] return (self.name, self.requirement) @classmethod @lru_cache() def from_string(cls, line): - # type: (Text) -> BaseRequirement + # type: (S) -> BaseRequirement line = line.strip() req = init_requirement(line) return cls.from_req(req) @@ -406,11 +408,11 @@ class BaseRequirement(object): @attr.s(slots=True, frozen=True) class Extra(object): - name = attr.ib(default=None, cmp=True) # type: Text + name = attr.ib(default=None, cmp=True) # type: STRING_TYPE requirements = attr.ib(factory=frozenset, cmp=True, type=frozenset) def __str__(self): - # type: () -> Text + # type: () -> S return "{0}: {{{1}}}".format(self.section, ", ".join([r.name for r in self.requirements])) def add(self, req): @@ -420,18 +422,18 @@ class Extra(object): return self def as_dict(self): - # type: () -> Dict[Text, Tuple[RequirementType, ...]] + # type: () -> Dict[S, Tuple[RequirementType, ...]] return {self.name: tuple([r.requirement for r in self.requirements])} @attr.s(slots=True, cmp=True, hash=True) class SetupInfo(object): - name = attr.ib(default=None, cmp=True) # type: Text - base_dir = attr.ib(default=None, cmp=True, hash=False) # type: Text - version = attr.ib(default=None, cmp=True) # type: Text + name = attr.ib(default=None, cmp=True) # type: STRING_TYPE + base_dir = attr.ib(default=None, cmp=True, hash=False) # type: STRING_TYPE + version = attr.ib(default=None, cmp=True) # type: STRING_TYPE _requirements = attr.ib(type=frozenset, factory=frozenset, cmp=True, hash=True) build_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) - build_backend = attr.ib(cmp=True) # type: Text + build_backend = attr.ib(cmp=True) # type: STRING_TYPE setup_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None, cmp=True) _extras_requirements = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) @@ -440,20 +442,21 @@ class SetupInfo(object): pyproject = attr.ib(type=Path, default=None, cmp=True, hash=False) ireq = attr.ib(default=None, cmp=True, hash=False) # type: Optional[InstallRequirement] extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict, cmp=False, hash=False) - metadata = attr.ib(default=None) # type: Optional[Tuple[Text]] + metadata = attr.ib(default=None) # type: Optional[Tuple[STRING_TYPE]] @build_backend.default def get_build_backend(self): + # type: () -> S return get_default_pyproject_backend() @property def requires(self): - # type: () -> Dict[Text, RequirementType] + # type: () -> Dict[S, RequirementType] return {req.name: req.requirement for req in self._requirements} @property def extras(self): - # type: () -> Dict[Text, Optional[Any]] + # type: () -> Dict[S, Optional[Any]] extras_dict = {} extras = set(self._extras_requirements) for section, deps in extras: @@ -465,7 +468,7 @@ class SetupInfo(object): @classmethod def get_setup_cfg(cls, setup_cfg_path): - # type: (Text) -> Dict[Text, Union[Text, None, Set[BaseRequirement], List[Text], Tuple[Text, Tuple[BaseRequirement]]]] + # type: (S) -> Dict[S, Union[S, None, Set[BaseRequirement], List[S], Tuple[S, Tuple[BaseRequirement]]]] if os.path.exists(setup_cfg_path): default_opts = { "metadata": {"name": "", "version": ""}, @@ -514,7 +517,8 @@ class SetupInfo(object): @property def egg_base(self): - base = None # type: Optional[Text] + # type: () -> S + base = None # type: Optional[STRING_TYPE] if self.setup_py.exists(): base = self.setup_py.parent elif self.pyproject.exists(): @@ -637,7 +641,7 @@ class SetupInfo(object): return config def build_wheel(self): - # type: () -> Text + # type: () -> S if not self.pyproject.exists(): build_requires = ", ".join(['"{0}"'.format(r) for r in self.build_requires]) self.pyproject.write_text(u""" @@ -653,7 +657,7 @@ build-backend = "{1}" # noinspection PyPackageRequirements def build_sdist(self): - # type: () -> Text + # type: () -> S if not self.pyproject.exists(): build_requires = ", ".join(['"{0}"'.format(r) for r in self.build_requires]) self.pyproject.write_text(u""" @@ -668,7 +672,7 @@ build-backend = "{1}" ) def build(self): - # type: () -> Optional[Text] + # type: () -> None dist_path = None try: dist_path = self.build_wheel() @@ -686,9 +690,10 @@ build-backend = "{1}" self.get_egg_metadata() if not self.metadata or not self.name: self.run_setup() + return None def reload(self): - # type: () -> Dict[Text, Any] + # type: () -> Dict[S, Any] """ Wipe existing distribution info metadata for rebuilding. """ @@ -700,13 +705,13 @@ build-backend = "{1}" self.get_info() def get_metadata_from_wheel(self, wheel_path): - # type: (Text) -> Dict[Any, Any] + # type: (S) -> Dict[Any, Any] metadata_dict = get_metadata_from_wheel(wheel_path) if metadata_dict: self.populate_metadata(metadata_dict) def get_egg_metadata(self, metadata_dir=None, metadata_type=None): - # type: (Optional[Text], Optional[Text]) -> None + # type: (Optional[AnyStr], Optional[AnyStr]) -> None package_indicators = [self.pyproject, self.setup_py, self.setup_cfg] # if self.setup_py is not None and self.setup_py.exists(): metadata_dirs = [] @@ -776,7 +781,7 @@ build-backend = "{1}" self.build_requires = ("setuptools", "wheel") def get_info(self): - # type: () -> Dict[Text, Any] + # type: () -> Dict[S, Any] if self.setup_cfg and self.setup_cfg.exists(): with cd(self.base_dir): self.parse_setup_cfg() @@ -800,7 +805,7 @@ build-backend = "{1}" return self.as_dict() def as_dict(self): - # type: () -> Dict[Text, Any] + # type: () -> Dict[S, Any] prop_dict = { "name": self.name, "version": self.version, @@ -829,7 +834,7 @@ build-backend = "{1}" @classmethod @lru_cache() def from_ireq(cls, ireq, subdir=None, finder=None): - # type: (InstallRequirement, Optional[Text], Optional[PackageFinder]) -> Optional[SetupInfo] + # type: (InstallRequirement, Optional[AnyStr], Optional[PackageFinder]) -> Optional[SetupInfo] import pip_shims.shims if not ireq.link: return @@ -891,7 +896,7 @@ build-backend = "{1}" @classmethod def create(cls, base_dir, subdirectory=None, ireq=None, kwargs=None): - # type: (Text, Optional[Text], Optional[InstallRequirement], Optional[Dict[Text, Text]]) -> Optional[SetupInfo] + # type: (AnyStr, Optional[AnyStr], Optional[InstallRequirement], Optional[Dict[AnyStr, AnyStr]]) -> Optional[SetupInfo] if not base_dir or base_dir is None: return diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 387997f3..000b9557 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -31,7 +31,7 @@ from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri from ..environment import MYPY_RUNNING if MYPY_RUNNING: - from typing import Union, Optional, List, Set, Any, TypeVar, Tuple, Sequence, Dict, Text + from typing import Union, Optional, List, Set, Any, TypeVar, Tuple, Sequence, Dict, Text, AnyStr, Match, Iterable from attr import _ValidatorType from packaging.requirements import Requirement as PackagingRequirement from pkg_resources import Requirement as PkgResourcesRequirement @@ -48,6 +48,8 @@ if MYPY_RUNNING: TOp = TypeVar("TOp", PkgResourcesOp, Op) MarkerTuple = Tuple[TVariable, TOp, TValue] TRequirement = Union[PackagingRequirement, PkgResourcesRequirement] + STRING_TYPE = Union[bytes, str, Text] + S = TypeVar("S", bytes, str, Text) HASH_STRING = " --hash={0}" @@ -69,7 +71,7 @@ DIRECT_URL_RE = re.compile(r"{0}\s?@\s?{1}".format(NAME_WITH_EXTRAS, URL)) def filter_none(k, v): - # type: (Text, Any) -> bool + # type: (AnyStr, Any) -> bool if v: return True return False @@ -81,7 +83,7 @@ def optional_instance_of(cls): def create_link(link): - # type: (Text) -> Link + # type: (AnyStr) -> Link if not isinstance(link, six.string_types): raise TypeError("must provide a string to instantiate a new link") @@ -90,7 +92,7 @@ def create_link(link): def get_url_name(url): - # type: (Text) -> Text + # type: (AnyStr) -> AnyStr """ Given a url, derive an appropriate name to use in a pipfile. @@ -104,7 +106,7 @@ def get_url_name(url): def init_requirement(name): - # type: (Text) -> TRequirement + # type: (AnyStr) -> TRequirement if not isinstance(name, six.string_types): raise TypeError("must supply a name to generate a requirement") @@ -118,7 +120,7 @@ def init_requirement(name): def extras_to_string(extras): - # type: (Sequence) -> Text + # type: (Iterable[S]) -> S """Turn a list of extras into a string""" if isinstance(extras, six.string_types): if extras.startswith("["): @@ -127,11 +129,11 @@ def extras_to_string(extras): extras = [extras] if not extras: return "" - return "[{0}]".format(",".join(sorted(set(extras)))) + return "[{0}]".format(",".join(sorted(set(extras)))) # type: ignore def parse_extras(extras_str): - # type: (Text) -> List + # type: (AnyStr) -> List[AnyStr] """ Turn a string of extras into a parsed extras list """ @@ -142,7 +144,7 @@ def parse_extras(extras_str): def specs_to_string(specs): - # type: (List[Union[Text, Specifier]]) -> Text + # type: (List[Union[STRING_TYPE, Specifier]]) -> AnyStr """ Turn a list of specifier tuples into a string """ @@ -153,20 +155,20 @@ def specs_to_string(specs): try: extras = ",".join(["".join(spec) for spec in specs]) except TypeError: - extras = ",".join(["".join(spec._spec) for spec in specs]) + extras = ",".join(["".join(spec._spec) for spec in specs]) # type: ignore return extras return "" def build_vcs_uri( - vcs, # type: Optional[Text] - uri, # type: Text - name=None, # type: Optional[Text] - ref=None, # type: Optional[Text] - subdirectory=None, # type: Optional[Text] - extras=None # type: Optional[List[Text]] + vcs, # type: Optional[S] + uri, # type: S + name=None, # type: Optional[S] + ref=None, # type: Optional[S] + subdirectory=None, # type: Optional[S] + extras=None # type: Optional[Iterable[S]] ): - # type: (...) -> Text + # type: (...) -> STRING_TYPE if extras is None: extras = [] vcs_start = "" @@ -187,25 +189,28 @@ def build_vcs_uri( def convert_direct_url_to_url(direct_url): - # type: (Text) -> Text + # type: (AnyStr) -> AnyStr """ Given a direct url as defined by *PEP 508*, convert to a :class:`~pip_shims.shims.Link` compatible URL by moving the name and extras into an **egg_fragment**. :param str direct_url: A pep-508 compliant direct url. :return: A reformatted URL for use with Link objects and :class:`~pip_shims.shims.InstallRequirement` objects. - :rtype: Text + :rtype: AnyStr """ - direct_match = DIRECT_URL_RE.match(direct_url) + direct_match = DIRECT_URL_RE.match(direct_url) # type: Optional[Match] if direct_match is None: url_match = URL_RE.match(direct_url) if url_match or is_valid_url(direct_url): return direct_url - match_dict = direct_match.groupdict() + match_dict = {} # type: Dict[STRING_TYPE, Union[Tuple[STRING_TYPE, ...], STRING_TYPE]] + if direct_match is not None: + match_dict = direct_match.groupdict() # type: ignore if not match_dict: raise ValueError("Failed converting value to normal URL, is it a direct URL? {0!r}".format(direct_url)) url_segments = [match_dict.get(s) for s in ("scheme", "host", "path", "pathsep")] - url = "".join([s for s in url_segments if s is not None]) + url = "" # type: STRING_TYPE + url = "".join([s for s in url_segments if s is not None]) # type: ignore new_url = build_vcs_uri( None, url, @@ -218,15 +223,15 @@ def convert_direct_url_to_url(direct_url): def convert_url_to_direct_url(url, name=None): - # type: (Text, Optional[Text]) -> Text + # type: (AnyStr, Optional[AnyStr]) -> AnyStr """ Given a :class:`~pip_shims.shims.Link` compatible URL, convert to a direct url as defined by *PEP 508* by extracting the name and extras from the **egg_fragment**. - :param Text url: A :class:`~pip_shims.shims.InstallRequirement` compliant URL. - :param Optiona[Text] name: A name to use in case the supplied URL doesn't provide one. + :param AnyStr url: A :class:`~pip_shims.shims.InstallRequirement` compliant URL. + :param Optiona[AnyStr] name: A name to use in case the supplied URL doesn't provide one. :return: A pep-508 compliant direct url. - :rtype: Text + :rtype: AnyStr :raises ValueError: Raised when the URL can't be parsed or a name can't be found. :raises TypeError: When a non-string input is provided. @@ -266,7 +271,7 @@ def convert_url_to_direct_url(url, name=None): def get_version(pipfile_entry): - # type: (Union[Text, Dict[Text, bool, List[Text]]]) -> Text + # type: (Union[STRING_TYPE, Dict[STRING_TYPE, Union[STRING_TYPE, bool, Iterable[STRING_TYPE]]]]) -> STRING_TYPE if str(pipfile_entry) == "{}" or is_star(pipfile_entry): return "" @@ -287,7 +292,7 @@ def strip_extras_markers_from_requirement(req): *extra == 'name'*, strip out the extras from the markers and return the cleaned requirement - :param PackagingRequirement req: A pacakaging requirement to clean + :param PackagingRequirement req: A packaging requirement to clean :return: A cleaned requirement :rtype: PackagingRequirement """ @@ -327,7 +332,7 @@ def _strip_extras_markers(marker): @lru_cache() def get_setuptools_version(): - # type: () -> Optional[Text] + # type: () -> Optional[STRING_TYPE] import pkg_resources setuptools_dist = pkg_resources.get_distribution( pkg_resources.Requirement("setuptools") @@ -336,7 +341,7 @@ def get_setuptools_version(): def get_default_pyproject_backend(): - # type: () -> Text + # type: () -> STRING_TYPE st_version = get_setuptools_version() if st_version is not None: parsed_st_version = parse_version(st_version) @@ -346,14 +351,14 @@ def get_default_pyproject_backend(): def get_pyproject(path): - # type: (Union[Text, Path]) -> Tuple[List[Text], Text] + # type: (Union[STRING_TYPE, Path]) -> Optional[Tuple[List[STRING_TYPE], STRING_TYPE]] """ Given a base path, look for the corresponding ``pyproject.toml`` file and return its build_requires and build_backend. - :param Text path: The root path of the project, should be a directory (will be truncated) + :param AnyStr path: The root path of the project, should be a directory (will be truncated) :return: A 2 tuple of build requirements and the build backend - :rtype: Tuple[List[Text], Text] + :rtype: Optional[Tuple[List[AnyStr], AnyStr]] """ if not path: @@ -390,11 +395,11 @@ def get_pyproject(path): else: requires = build_system.get("requires", ["setuptools>=40.8", "wheel"]) backend = build_system.get("build-backend", get_default_pyproject_backend()) - return (requires, backend) + return requires, backend def split_markers_from_line(line): - # type: (Text) -> Tuple[Text, Optional[Text]] + # type: (AnyStr) -> Tuple[AnyStr, Optional[AnyStr]] """Split markers from a dependency""" if not any(line.startswith(uri_prefix) for uri_prefix in SCHEME_LIST): marker_sep = ";" @@ -408,9 +413,10 @@ def split_markers_from_line(line): def split_vcs_method_from_uri(uri): - # type: (Text) -> Tuple[Optional[Text], Text] + # type: (AnyStr) -> Tuple[Optional[STRING_TYPE], STRING_TYPE] """Split a vcs+uri formatted uri into (vcs, uri)""" vcs_start = "{0}+" + vcs = None # type: Optional[STRING_TYPE] vcs = first([vcs for vcs in VCS_LIST if uri.startswith(vcs_start.format(vcs))]) if vcs: vcs, uri = uri.split("+", 1) @@ -418,14 +424,14 @@ def split_vcs_method_from_uri(uri): def split_ref_from_uri(uri): - # type: (Text) -> Tuple[Text, Optional[Text]] + # type: (AnyStr) -> Tuple[AnyStr, Optional[AnyStr]] """ Given a path or URI, check for a ref and split it from the path if it is present, returning a tuple of the original input and the ref or None. - :param Text uri: The path or URI to split + :param AnyStr uri: The path or URI to split :returns: A 2-tuple of the path or URI and the ref - :rtype: Tuple[Text, Optional[Text]] + :rtype: Tuple[AnyStr, Optional[AnyStr]] """ if not isinstance(uri, six.string_types): raise TypeError("Expected a string, received {0!r}".format(uri)) @@ -494,8 +500,8 @@ def _requirement_to_str_lowercase_name(requirement): modified to lowercase the dependency name. Previously, we were invoking the original Requirement.__str__ method and - lowercasing the entire result, which would lowercase the name, *and* other, - important stuff that should not be lowercased (such as the marker). See + lower-casing the entire result, which would lowercase the name, *and* other, + important stuff that should not be lower-cased (such as the marker). See this issue for more information: https://github.com/pypa/pipenv/issues/2113. """ @@ -818,12 +824,12 @@ def fix_requires_python_marker(requires_python): def normalize_name(pkg): - # type: (Text) -> Text + # type: (AnyStr) -> AnyStr """Given a package name, return its normalized, non-canonicalized form. - :param Text pkg: The name of a package + :param AnyStr pkg: The name of a package :return: A normalized package name - :rtype: Text + :rtype: AnyStr """ assert isinstance(pkg, six.string_types) @@ -831,12 +837,12 @@ def normalize_name(pkg): def get_name_variants(pkg): - # type: (Text) -> Set[Text] + # type: (STRING_TYPE) -> Set[STRING_TYPE] """ Given a packager name, get the variants of its name for both the canonicalized and "safe" forms. - :param Text pkg: The package to lookup + :param AnyStr pkg: The package to lookup :returns: A list of names. :rtype: Set """ diff --git a/pipenv/vendor/vistir/cmdparse.py b/pipenv/vendor/vistir/cmdparse.py index 07326c93..5a307f24 100644 --- a/pipenv/vendor/vistir/cmdparse.py +++ b/pipenv/vendor/vistir/cmdparse.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import itertools import re import shlex @@ -14,6 +15,12 @@ class ScriptEmptyError(ValueError): pass +def _quote_if_contains(value, pattern): + if next(re.finditer(pattern, value), None): + return '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', value)) + return value + + class Script(object): """Parse a script line (in Pipfile's [scripts] section). @@ -71,8 +78,7 @@ class Script(object): See also: https://docs.python.org/3/library/subprocess.html#converting-argument-sequence """ - return " ".join( - arg if not next(re.finditer(r'\s', arg), None) - else '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', arg)) - for arg in self._parts - ) + return " ".join(itertools.chain( + [_quote_if_contains(self.command, r'[\s^()]')], + (_quote_if_contains(arg, r'[\s^]') for arg in self.args), + )) From 199a14ac0a8ae98a5bb2a782c44ae79a7db5c826 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 1 Mar 2019 19:22:00 -0500 Subject: [PATCH 05/21] Incorporate fixes from feature/keep-outdated-peep Signed-off-by: Dan Ryan --- pipenv/core.py | 28 ++++++++++-------- pipenv/environment.py | 29 ++++++++++++++++--- pipenv/project.py | 3 +- pipenv/utils.py | 25 ++++++++++------ .../requirementslib/models/requirements.py | 8 ++--- 5 files changed, 62 insertions(+), 31 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 22ef34bc..e229db3b 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1047,12 +1047,8 @@ def do_lock( err=True, ) - deps = convert_deps_to_pip( - packages, project, r=False, include_index=True - ) - # Mutates the lockfile venv_resolve_deps( - deps, + packages, which=which, project=project, dev=is_dev, @@ -1296,6 +1292,10 @@ def pip_install( crayons.normal("Installing {0!r}".format(requirement.name), bold=True), err=True, ) + + if requirement: + ignore_hashes = True if not requirement.hashes else ignore_hashes + # Create files for hash mode. if write_to_tmpfile: if not requirements_dir: @@ -1305,14 +1305,13 @@ def pip_install( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, delete=False ) - f.write(vistir.misc.to_bytes(requirement.as_line())) + line = requirement.as_line(include_hashes=not ignore_hashes) + f.write(vistir.misc.to_bytes(line)) r = f.name f.close() if requirement and requirement.vcs: # Install dependencies when a package is a non-editable VCS dependency. - if not requirement.editable: - no_deps = False # Don't specify a source directory when using --system. if not allow_global and ("PIP_SRC" not in os.environ): src.extend(["--src", "{0}".format(project.virtualenv_src_location)]) @@ -1363,6 +1362,7 @@ def pip_install( # Install dependencies when a package is a VCS dependency. if requirement and requirement.vcs: + ignore_hashes = True # Don't specify a source directory when using --system. src_dir = None if "PIP_SRC" in os.environ: @@ -1421,17 +1421,19 @@ def pip_install( install_reqs = requirement.as_line(**line_kwargs) if requirement.editable and install_reqs[0].startswith("-e "): req, install_reqs = install_reqs[0], install_reqs[1:] + possible_hashes = install_reqs[:] editable_opt, req = req.split(" ", 1) install_reqs = [editable_opt, req] + install_reqs - if not any(item.startswith("--hash") for item in install_reqs): - ignore_hashes = True + + # hashes must be passed via a file + ignore_hashes = True elif r: install_reqs = ["-r", r] with open(r) as f: if "--hash" not in f.read(): ignore_hashes = True else: - ignore_hashes = True if not requirement.hashes else False + ignore_hashes = True install_reqs = requirement.as_line(as_list=True, include_hashes=not ignore_hashes) if not requirement.markers: install_reqs = [escape_cmd(r) for r in install_reqs] @@ -1454,8 +1456,10 @@ def pip_install( if not ignore_hashes: pip_command.append("--require-hashes") if not use_pep517: + from .vendor.packaging.version import parse as parse_version pip_command.append("--no-build-isolation") - pip_command.append("--no-use-pep517") + if project.environment.pip_version >= parse_version("19.0"): + pip_command.append("--no-use-pep517") if environments.is_verbose(): click.echo("$ {0}".format(pip_command), err=True) cache_dir = vistir.compat.Path(PIPENV_CACHE_DIR) diff --git a/pipenv/environment.py b/pipenv/environment.py index 9e75b7d8..602859e2 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -188,12 +188,14 @@ class Environment(object): @cached_property def sys_path(self): - """The system path inside the environment + """ + The system path inside the environment :return: The :data:`sys.path` from the environment :rtype: list """ + from .vendor.vistir.compat import JSONDecodeError current_executable = vistir.compat.Path(sys.executable).as_posix() if not self.python or self.python == current_executable: return sys.path @@ -201,12 +203,16 @@ class Environment(object): return sys.path cmd_args = [self.python, "-c", "import json, sys; print(json.dumps(sys.path))"] path, _ = vistir.misc.run(cmd_args, return_object=False, nospin=True, block=True, combine_stderr=False, write_to_stdout=False) - path = json.loads(path.strip()) + try: + path = json.loads(path.strip()) + except JSONDecodeError: + path = sys.path return path @cached_property def sys_prefix(self): - """The prefix run inside the context of the environment + """ + The prefix run inside the context of the environment :return: The python prefix inside the environment :rtype: :data:`sys.prefix` @@ -241,8 +247,23 @@ class Environment(object): return "purelib", purelib return "platlib", self.paths["platlib"] + @property + def pip_version(self): + """ + Get the pip version in the environment. Useful for knowing which args we can use + when installing. + """ + from .vendor.packaging.version import parse as parse_version + pip = next(iter( + pkg for pkg in self.get_installed_packages() if pkg.key == "pip" + ), None) + if pip is not None: + pip_version = parse_version(pip.version) + return parse_version("18.0") + def get_distributions(self): - """Retrives the distributions installed on the library path of the environment + """ + Retrives the distributions installed on the library path of the environment :return: A set of distributions found on the library path :rtype: iterator diff --git a/pipenv/project.py b/pipenv/project.py index 8e209638..eb802cbb 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -537,7 +537,8 @@ class Project(object): @property def build_requires(self): - return self._build_system.get("requires", []) + return self._build_system.get("requires", ["setuptools>=40.8.0", "wheel"]) + @property def build_backend(self): diff --git a/pipenv/utils.py b/pipenv/utils.py index aec4ee96..ef290e54 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -900,6 +900,7 @@ def venv_resolve_deps( from .vendor.vistir.compat import Path, JSONDecodeError from .vendor.vistir.path import create_tracked_tempdir from . import resolver + from ._compat import decode_for_output import json results = [] @@ -908,12 +909,7 @@ def venv_resolve_deps( if not deps: if not project.pipfile_exists: return None - # This is a requirementslib pipfile instance which provides `Requirement` instances - # rather than simply locked dependencies in a lockfile format - deps = convert_deps_to_pip( - project.parsed_pipfile.get(pipfile_section, {}), project=project, - r=False, include_index=True - ) + deps = project.parsed_pipfile.get(pipfile_section, {}) if not deps: return None @@ -922,7 +918,6 @@ def venv_resolve_deps( if not lockfile: lockfile = project._lockfile req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") - constraints = set(deps) cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() @@ -933,16 +928,28 @@ def venv_resolve_deps( cmd.append("--clear") if allow_global: cmd.append("--system") + if dev: + cmd.append("--dev") with temp_environ(): os.environ.update({fs_str(k): fs_str(val) for k, val in os.environ.items()}) - os.environ["PIPENV_PACKAGES"] = str("\n".join(constraints)) if pypi_mirror: os.environ["PIPENV_PYPI_MIRROR"] = str(pypi_mirror) os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") os.environ["PIPENV_SITE_DIR"] = get_pipenv_sitedir() - with create_spinner(text=fs_str("Locking...")) as sp: + with create_spinner(text=decode_for_output("Locking...")) as sp: + # This conversion is somewhat slow on local and file-type requirements since + # we now download those requirements / make temporary folders to perform + # dependency resolution on them, so we are including this step inside the + # spinner context manager for the UX improvement + sp.write(decode_for_output("Building requirements...")) + deps = convert_deps_to_pip( + deps, project, r=False, include_index=True + ) + constraints = set(deps) + os.environ["PIPENV_PACKAGES"] = str("\n".join(constraints)) + sp.write(decode_for_output("Resolving dependencies...")) c = resolve(cmd, sp) results = c.out.strip() sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index d6beb085..d57e4fff 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -2078,16 +2078,14 @@ class VCSRequirement(FileRequirement): path = pip_shims.shims.url_to_path(self.uri) if path and os.path.exists(path): checkout_dir = os.path.abspath(path) - return vistir.compat.fs_encode(checkout_dir) + return checkout_dir if src_dir is not None: checkout_dir = os.path.join( os.path.abspath(src_dir), self.name ) - mkdir_p(vistir.compat.fs_encode(src_dir)) + mkdir_p(src_dir) return checkout_dir - return vistir.compat.fs_encode( - os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) - ) + return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) def get_vcs_repo(self, src_dir=None, checkout_dir=None): # type: (Optional[STRING_TYPE], STRING_TYPE) -> VCSRepository From be3c7714ee374ad0311becccbe72fdbe54414767 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 2 Mar 2019 14:20:30 -0500 Subject: [PATCH 06/21] Cut in new cursor implementation - Fix some errors in exception handler - Update yaspin patch - Fix spinner Signed-off-by: Dan Ryan --- pipenv/core.py | 4 ++-- pipenv/exceptions.py | 11 +++++++++-- pipenv/utils.py | 8 ++++---- pipenv/vendor/yaspin/core.py | 6 +++--- .../patches/vendor/yaspin-signal-handling.patch | 6 +++--- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index e229db3b..7daff2be 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -11,7 +11,7 @@ import warnings import click import six import urllib3.util as urllib3_util -import vistir +from pipenv.vendor import vistir import click_completion import crayons @@ -1273,7 +1273,7 @@ def pip_install( use_pep517=True ): from pipenv.patched.notpip._internal import logger as piplogger - from .utils import Mapping + from .vendor.vistir.compat import Mapping from .vendor.urllib3.util import parse_url src = [] write_to_tmpfile = False diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 4e9ce95f..47dfc718 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -179,7 +179,8 @@ class SystemUsageError(PipenvOptionsError): crayons.red("Warning", bold=True) ), ] - message = crayons.blue("See also: {0}".format(crayons.white("-deploy flag."))) + if message is None: + message = crayons.blue("See also: {0}".format(crayons.white("--deploy flag."))) super(SystemUsageError, self).__init__(option_name, message=message, ctx=ctx, extra=extra, **kwargs) @@ -248,8 +249,14 @@ class UninstallError(PipenvException): class InstallError(PipenvException): def __init__(self, package, **kwargs): - message = "{0} {1}".format( + package_message = "" + if package is not None: + package_message = crayons.normal("Couldn't install package {0}\n".format( + crayons.white(package, bold=True) + )) + message = "{0} {1} {2}".format( crayons.red("ERROR:", bold=True), + package_message, crayons.yellow("Package installation failed...") ) extra = kwargs.pop("extra", []) diff --git a/pipenv/utils.py b/pipenv/utils.py index ef290e54..75f89299 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -754,19 +754,19 @@ def actually_resolve_deps( @contextlib.contextmanager def create_spinner(text, nospin=None, spinner_name=None): - import vistir.spin + from .vendor.vistir import spin + from .vendor.vistir.misc import fs_str if not spinner_name: spinner_name = environments.PIPENV_SPINNER if nospin is None: nospin = environments.PIPENV_NOSPIN - with vistir.spin.create_spinner( + with spin.create_spinner( spinner_name=spinner_name, - start_text=vistir.compat.fs_str(text), + start_text=fs_str(text), nospin=nospin, write_to_stdout=False ) as sp: yield sp - def resolve(cmd, sp): import delegator from .cmdparse import Script diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index 06b8b621..dcaa9e42 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -17,7 +17,7 @@ import threading import time import colorama -import cursor +from pipenv.vendor.vistir import cursor from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str @@ -530,11 +530,11 @@ class Yaspin(object): @staticmethod def _hide_cursor(): - cursor.hide() + cursor.hide_cursor() @staticmethod def _show_cursor(): - cursor.show() + cursor.show_cursor() @staticmethod def _clear_line(): diff --git a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch index a1d27cd3..511e782a 100644 --- a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch +++ b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch @@ -7,7 +7,7 @@ index d01fb98e..06b8b621 100644 import time +import colorama -+import cursor ++from pipenv.vendor.vistir import cursor + from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str @@ -48,13 +48,13 @@ index d01fb98e..06b8b621 100644 def _hide_cursor(): - sys.stdout.write("\033[?25l") - sys.stdout.flush() -+ cursor.hide() ++ cursor.hide_cursor() @staticmethod def _show_cursor(): - sys.stdout.write("\033[?25h") - sys.stdout.flush() -+ cursor.show() ++ cursor.show_cursor() @staticmethod def _clear_line(): From 57b266f256f9b74aaf6386c4467fe05af51e1c5f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 2 Mar 2019 17:13:20 -0500 Subject: [PATCH 07/21] Update requirementslib to include extras in parser Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/requirements.py | 7 +++++++ pipenv/vendor/requirementslib/models/setup_info.py | 3 +++ 2 files changed, 10 insertions(+) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index d57e4fff..bf207cf9 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1289,6 +1289,13 @@ class FileRequirement(object): deps.update(setup_info.get("requires", {})) setup_deps.extend(setup_info.get("setup_requires", [])) build_deps.extend(setup_info.get("build_requires", [])) + if self.extras and self.setup_info.extras: + for dep in self.extras: + if dep not in self.setup_info.extras: + continue + extras_list = self.setup_info.extras.get(dep, []) # type: ignore + for req_instance in extras_list: # type: ignore + deps[req_instance.key] = req_instance if self.pyproject_requires: build_deps.extend(list(self.pyproject_requires)) setup_deps = list(set(setup_deps)) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index f278afd2..c8f9e93f 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -560,6 +560,9 @@ class SetupInfo(object): if extra in self.extras: extras_tuple = tuple([BaseRequirement.from_req(req) for req in self.extras[extra]]) self._extras_requirements += ((extra, extras_tuple),) + self._requirements = frozenset( + set(self._requirements) | set(list(extras_tuple)) + ) def run_setup(self): # type: () -> None From e238c840ed666d0aefbf54c371aa65bf1a9bdbbf Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 3 Mar 2019 00:05:14 -0500 Subject: [PATCH 08/21] Only resolve requirements with markers we can evaluate Signed-off-by: Dan Ryan --- news/3298.bugfix.rst | 3 + news/3298.feature.rst | 5 + news/3298.vendor.rst | 23 + pipenv/utils.py | 15 +- .../requirementslib/models/requirements.py | 538 ++++++++++++------ .../requirementslib/models/setup_info.py | 255 ++++++--- 6 files changed, 582 insertions(+), 257 deletions(-) create mode 100644 news/3298.bugfix.rst create mode 100644 news/3298.feature.rst create mode 100644 news/3298.vendor.rst diff --git a/news/3298.bugfix.rst b/news/3298.bugfix.rst new file mode 100644 index 00000000..1e27283a --- /dev/null +++ b/news/3298.bugfix.rst @@ -0,0 +1,3 @@ +- Fixed a bug which caused failures in warning reporting when running pipenv inside a virtualenv under some circumstances. + +Fixed a bug with package discovery when running ``pipenv clean``. diff --git a/news/3298.feature.rst b/news/3298.feature.rst new file mode 100644 index 00000000..8aaf4e23 --- /dev/null +++ b/news/3298.feature.rst @@ -0,0 +1,5 @@ +- Added full support for resolution of all dependency types including direct URLs, zip archives, tarballs, etc. + +Improved error handling and formatting. + +Introduced improved cross platform stream wrappers for better ``stdout`` and ``stderr`` consistency. diff --git a/news/3298.vendor.rst b/news/3298.vendor.rst new file mode 100644 index 00000000..f829fcf0 --- /dev/null +++ b/news/3298.vendor.rst @@ -0,0 +1,23 @@ +- Updated vendored dependencies: + + - **certifi**: ``2018.10.15`` => ``2018.11.29`` + - **cached_property**: ``1.4.3`` => ``1.5.1`` + - **colorama**: ``0.3.9`` => ``0.4.1`` + - **idna**: ``2.7`` => ``2.8`` + - **packaging**: ``18.0`` => ``19.0`` + - **pathlib2**: ``2.3.2`` => ``2.3.3`` + - **pep517**: ``(new)`` => ``0.5.0`` + - **pipdeptree**: ``0.13.0`` => ``0.13.1`` + - **pyparsing**: ``2.2.2`` => ``2.3.1`` + - **python-dotenv**: ``0.9.1`` => ``0.10.1`` + - **pythonfinder**: ``1.1.10`` => ``1.2.0`` + - **pytoml**: ``(new)`` => ``0.1.20`` + - **requests**: ``2.20.1`` => ``2.21.0`` + - **requirementslib**: ``1.3.3`` => ``1.4.0`` + - **shellingham**: ``1.2.7`` => ``1.2.8`` + - **six**: ``1.11.0`` => ``1.12.0`` + - **tomlkit**: ``0.5.2`` => ``0.5.3`` + - **urllib3**: ``1.24`` => ``1.24.1`` + - **vistir**: ``0.3.0`` => ``0.3.1`` + +Removed vendored dependency **cursor**. diff --git a/pipenv/utils.py b/pipenv/utils.py index 75f89299..3771f61f 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -330,6 +330,7 @@ class Resolver(object): ): # type: (...) -> Tuple[Requirement, Dict[str, str], Dict[str, str]] from .vendor.requirementslib.models.requirements import Requirement + from .exceptions import ResolutionFailure if index_lookup is None: index_lookup = {} if markers_lookup is None: @@ -342,7 +343,10 @@ class Resolver(object): if indexes: url = indexes[0] line = " ".join(remainder) - req = Requirement.from_line(line) + try: + req = Requirement.from_line(line) + except ValueError: + raise ResolutionFailure("Failed to resolve requirement from line: {0!s}".format(line)) if url: index_lookup[req.normalized_name] = project.get_source( url=url, refresh=True).get("name") @@ -386,7 +390,14 @@ class Resolver(object): continue line = _requirement_to_str_lowercase_name(r) new_req, _, _ = cls.parse_line(line) - new_constraints, new_lock = cls.get_deps_from_req(new_req) + if r.marker and not r.marker.evaluate(): + new_constraints = {} + _, new_entry = req.pipfile_entry + new_lock = { + pep_423_name(new_req.normalized_name): new_entry + } + else: + new_constraints, new_lock = cls.get_deps_from_req(new_req) locked_deps.update(new_lock) constraints |= new_constraints else: diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index bf207cf9..b60f8ca5 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -7,25 +7,28 @@ import copy import hashlib import os import sys - -from distutils.sysconfig import get_python_lib from contextlib import contextmanager +from distutils.sysconfig import get_python_lib from functools import partial import attr import pip_shims import six import vistir - -from first import first from cached_property import cached_property +from first import first from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement -from packaging.specifiers import Specifier, SpecifierSet, LegacySpecifier, InvalidSpecifier +from packaging.specifiers import ( + InvalidSpecifier, + LegacySpecifier, + Specifier, + SpecifierSet, +) from packaging.utils import canonicalize_name from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote -from vistir.compat import Path, FileNotFoundError, lru_cache, Mapping +from vistir.compat import FileNotFoundError, Mapping, Path, lru_cache from vistir.contextmanagers import temp_path from vistir.misc import dedup from vistir.path import ( @@ -33,30 +36,28 @@ from vistir.path import ( get_converted_relative_path, is_file_url, is_valid_url, + mkdir_p, normalize_path, - mkdir_p ) -from ..exceptions import RequirementError -from ..utils import ( - VCS_LIST, - is_installable_file, - is_vcs, - add_ssh_scheme_to_git_uri, - strip_ssh_from_git_uri, - get_setup_paths -) from .setup_info import SetupInfo, _prepare_wheel_building_kwargs from .utils import ( + DIRECT_URL_RE, HASH_STRING, + URL_RE, build_vcs_uri, + convert_direct_url_to_url, + create_link, extras_to_string, filter_none, format_requirement, + get_default_pyproject_backend, + get_pyproject, get_version, init_requirement, is_pinned_requirement, make_install_requirement, + normalize_name, parse_extras, specs_to_string, split_markers_from_line, @@ -65,25 +66,48 @@ from .utils import ( validate_path, validate_specifiers, validate_vcs, - normalize_name, - create_link, - get_pyproject, - convert_direct_url_to_url, - URL_RE, - DIRECT_URL_RE, - get_default_pyproject_backend +) +from ..environment import MYPY_RUNNING +from ..exceptions import RequirementError +from ..utils import ( + VCS_LIST, + add_ssh_scheme_to_git_uri, + get_setup_paths, + is_installable_file, + is_vcs, + strip_ssh_from_git_uri, ) -from ..environment import MYPY_RUNNING - if MYPY_RUNNING: - from typing import Optional, TypeVar, List, Dict, Union, Any, Tuple, Set, AnyStr, Text, Generator, FrozenSet - from pip_shims.shims import Link, InstallRequirement, PackageFinder, InstallationCandidate - RequirementType = TypeVar('RequirementType', covariant=True, bound=PackagingRequirement) + from typing import ( + Optional, + TypeVar, + List, + Dict, + Union, + Any, + Tuple, + Set, + AnyStr, + Text, + Generator, + FrozenSet, + ) + from pip_shims.shims import ( + Link, + InstallRequirement, + PackageFinder, + InstallationCandidate, + ) + + RequirementType = TypeVar( + "RequirementType", covariant=True, bound=PackagingRequirement + ) F = TypeVar("F", "FileRequirement", "VCSRequirement", covariant=True) from six.moves.urllib.parse import SplitResult from .vcs import VCSRepository from .dependencies import AbstractDependency + NON_STRING_ITERABLE = Union[List, Set, Tuple] STRING_TYPE = Union[str, bytes, Text] S = TypeVar("S", bytes, str, Text) @@ -103,7 +127,7 @@ class Line(object): # type: (AnyStr, Optional[Union[List[S], Set[S], Tuple[S, ...]]]) -> None self.editable = False # type: bool if line.startswith("-e "): - line = line[len("-e "):] + line = line[len("-e ") :] self.editable = True self.extras = () # type: Tuple[STRING_TYPE, ...] if extras is not None: @@ -140,9 +164,16 @@ class Line(object): self.parse() def __hash__(self): - return hash(( - self.editable, self.line, self.markers, tuple(self.extras), - tuple(self.hashes), self.vcs, self.ireq) + return hash( + ( + self.editable, + self.line, + self.markers, + tuple(self.extras), + tuple(self.hashes), + self.vcs, + self.ireq, + ) ) def __repr__(self): @@ -163,7 +194,7 @@ class Line(object): def split_hashes(cls, line): # type: (S) -> Tuple[S, List[S]] if "--hash" not in line: - return line, [] + return line, [] split_line = line.split() line_parts = [] # type: List[S] hashes = [] # type: List[S] @@ -200,11 +231,16 @@ class Line(object): line = "" # type: STRING_TYPE if self.is_file or self.is_url and not self.is_vcs: scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri" - local_line = next(iter([ - os.path.dirname(os.path.abspath(f)) for f in [ - self.setup_py, self.setup_cfg, self.pyproject_toml - ] if f is not None - ]), None) + local_line = next( + iter( + [ + os.path.dirname(os.path.abspath(f)) + for f in [self.setup_py, self.setup_cfg, self.pyproject_toml] + if f is not None + ] + ), + None, + ) if local_line and self.extras: local_line = "{0}{1}".format(local_line, extras_to_string(self.extras)) line = local_line if local_line is not None else self.line @@ -279,12 +315,16 @@ class Line(object): for req in (self.ireq, self.requirement): if req is not None and getattr(req, "specifier", None): options.append(req.specifier) - specifier = next(iter(spec for spec in options if spec is not None), None) # type: Optional[Union[Specifier, SpecifierSet]] + specifier = next( + iter(spec for spec in options if spec is not None), None + ) # type: Optional[Union[Specifier, SpecifierSet]] spec_string = None # type: Optional[STRING_TYPE] if specifier is not None: spec_string = specs_to_string(specifier) - elif specifier is None and not self.is_named and ( - self._setup_info is not None and self._setup_info.version + elif ( + specifier is None + and not self.is_named + and (self._setup_info is not None and self._setup_info.version) ): spec_string = "=={0}".format(self._setup_info.version) if spec_string: @@ -312,7 +352,12 @@ class Line(object): # TODO: Should we include versions for VCS dependencies? IS there a reason not # to? For now we are using hashes as the equivalent to pin # note: we need versions for direct dependencies at the very least - if self.is_file or self.is_url or self.is_path or (self.is_vcs and not self.editable): + if ( + self.is_file + or self.is_url + or self.is_path + or (self.is_vcs and not self.editable) + ): if self.specifier is not None: specifier = self.specifier if not isinstance(specifier, SpecifierSet): @@ -359,7 +404,11 @@ class Line(object): self._requirement = init_requirement(canonicalize_name(self.name)) if self.is_file or self.is_url and self._requirement is not None: self._requirement.url = self.url - if self._requirement and self._requirement.specifier and not self._requirement.specs: + if ( + self._requirement + and self._requirement.specifier + and not self._requirement.specs + ): specs = self.get_requirement_specs(self._requirement.specifier) self._requirement.specs = specs return self._requirement @@ -373,7 +422,9 @@ class Line(object): base_path = self.base_path if base_path is None: return - setup_paths = get_setup_paths(base_path, subdirectory=self.subdirectory) # type: Dict[STRING_TYPE, Optional[STRING_TYPE]] + setup_paths = get_setup_paths( + base_path, subdirectory=self.subdirectory + ) # type: Dict[STRING_TYPE, Optional[STRING_TYPE]] self._setup_py = setup_paths.get("setup_py") self._setup_cfg = setup_paths.get("setup_cfg") self._pyproject_toml = setup_paths.get("pyproject_toml") @@ -382,7 +433,9 @@ class Line(object): def pyproject_requires(self): # type: () -> Optional[Tuple[STRING_TYPE, ...]] if self._pyproject_requires is None and self.pyproject_toml is not None: - pyproject_requires, pyproject_backend = get_pyproject(self.path) # type: ignore + pyproject_requires, pyproject_backend = get_pyproject( + self.path + ) # type: ignore if pyproject_requires: self._pyproject_requires = tuple(pyproject_requires) self._pyproject_backend = pyproject_backend @@ -392,11 +445,15 @@ class Line(object): def pyproject_backend(self): # type: () -> Optional[STRING_TYPE] if self._pyproject_requires is None and self.pyproject_toml is not None: - pyproject_requires, pyproject_backend = get_pyproject(self.path) # type: ignore + pyproject_requires, pyproject_backend = get_pyproject( + self.path + ) # type: ignore if not pyproject_backend and self.setup_cfg is not None: setup_dict = SetupInfo.get_setup_cfg(self.setup_cfg) pyproject_backend = get_default_pyproject_backend() - pyproject_requires = setup_dict.get("build_requires", ["setuptools", "wheel"]) # type: ignore + pyproject_requires = setup_dict.get( + "build_requires", ["setuptools", "wheel"] + ) # type: ignore self._pyproject_requires = tuple(pyproject_requires) self._pyproject_backend = pyproject_backend @@ -584,17 +641,22 @@ class Line(object): def is_url(self): # type: () -> bool url = self.get_url() - if (is_valid_url(url) or is_file_url(url)): + if is_valid_url(url) or is_file_url(url): return True return False @property def is_path(self): # type: () -> bool - if self.path and ( - self.path.startswith(".") or os.path.isabs(self.path) or - os.path.exists(self.path) - ) and is_installable_file(self.path): + if ( + self.path + and ( + self.path.startswith(".") + or os.path.isabs(self.path) + or os.path.exists(self.path) + ) + and is_installable_file(self.path) + ): return True elif (os.path.exists(self.line) and is_installable_file(self.line)) or ( os.path.exists(self.get_url()) and is_installable_file(self.get_url()) @@ -614,11 +676,14 @@ class Line(object): @property def is_file(self): # type: () -> bool - if self.is_path or ( - is_file_url(self.get_url()) and is_installable_file(self.get_url()) - ) or ( - self._parsed_url and self._parsed_url.scheme == "file" and - is_installable_file(urllib_parse.urlunparse(self._parsed_url)) + if ( + self.is_path + or (is_file_url(self.get_url()) and is_installable_file(self.get_url())) + or ( + self._parsed_url + and self._parsed_url.scheme == "file" + and is_installable_file(urllib_parse.urlunparse(self._parsed_url)) + ) ): return True return False @@ -691,9 +756,12 @@ class Line(object): def _get_vcsrepo(self): # type: () -> Optional[VCSRepository] from .vcs import VCSRepository + checkout_directory = self.wheel_kwargs["src_dir"] # type: ignore if self.name is not None: - checkout_directory = os.path.join(checkout_directory, self.name) # type: ignore + checkout_directory = os.path.join( + checkout_directory, self.name + ) # type: ignore vcsrepo = VCSRepository( url=self.link.url, name=self.name, @@ -702,10 +770,7 @@ class Line(object): vcs_type=self.vcs, subdirectory=self.subdirectory, ) - if not ( - self.link.scheme.startswith("file") and - self.editable - ): + if not (self.link.scheme.startswith("file") and self.editable): vcsrepo.obtain() return vcsrepo @@ -729,8 +794,10 @@ class Line(object): with temp_path(): sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)] setupinfo = SetupInfo.create( - repo.checkout_directory, ireq=ireq, subdirectory=self.subdirectory, - kwargs=wheel_kwargs + repo.checkout_directory, + ireq=ireq, + subdirectory=self.subdirectory, + kwargs=wheel_kwargs, ) self._setup_info = setupinfo self._setup_info.reload() @@ -767,6 +834,7 @@ class Line(object): if not self.is_wheel: pass from pip_shims.shims import Wheel + _wheel = Wheel(self.link.filename) name = _wheel.name version = _wheel.version @@ -791,7 +859,9 @@ class Line(object): try: self._requirement = init_requirement(self.line) except Exception: - raise RequirementError("Failed parsing requirement from {0!r}".format(self.line)) + raise RequirementError( + "Failed parsing requirement from {0!r}".format(self.line) + ) name = self._requirement.name if not self._specifier and self._requirement and self._requirement.specifier: self._specifier = specs_to_string(self._requirement.specifier) @@ -853,7 +923,7 @@ class Line(object): ref=self.ref, subdirectory=self.subdirectory, extras=self.extras, - name=self.name + name=self.name, ) if vcs_uri: self._requirement.link = create_link(vcs_uri) @@ -916,7 +986,9 @@ class Line(object): def parse_link(self): # type: () -> None if self.is_file or self.is_url or self.is_vcs: - vcs, prefer, relpath, path, uri, link = FileRequirement.get_link_from_line(self.line) + vcs, prefer, relpath, path, uri, link = FileRequirement.get_link_from_line( + self.line + ) ref = None if link is not None and "@" in unquote(link.path) and uri is not None: uri, _, ref = unquote(uri).rpartition("@") @@ -944,10 +1016,14 @@ class Line(object): self._link = link if (self.is_direct_url or vcs) and self.name is not None and vcs is not None: self._link = create_link( - build_vcs_uri(vcs=vcs, uri=link_url, ref=ref, - extras=self.extras, name=self.name, - subdirectory=link.subdirectory_fragment - ) + build_vcs_uri( + vcs=vcs, + uri=link_url, + ref=ref, + extras=self.extras, + name=self.name, + subdirectory=link.subdirectory_fragment, + ) ) else: self._link = link @@ -1015,9 +1091,14 @@ class Line(object): link = create_link(line) line = link.url_without_fragment line, _ = split_ref_from_uri(line) - if (is_vcs(line) or (is_valid_url(line) and ( - not is_file_url(line) or is_installable_file(line))) - or is_installable_file(line)): + if ( + is_vcs(line) + or ( + is_valid_url(line) + and (not is_file_url(line) or is_installable_file(line)) + ) + or is_installable_file(line) + ): return True return False @@ -1027,7 +1108,9 @@ class Line(object): self.line, self.markers = split_markers_from_line(self.line) self.parse_extras() self.line = self.line.strip('"').strip("'").strip() - if self.line.startswith("git+file:/") and not self.line.startswith("git+file:///"): + if self.line.startswith("git+file:/") and not self.line.startswith( + "git+file:///" + ): self.line = self.line.replace("git+file:/", "git+file:///") self.parse_markers() if self.is_file_url: @@ -1086,7 +1169,7 @@ class NamedRequirement(object): "version": specifiers, "req": req, "parsed_line": parsed_line, - "extras": None + "extras": None, } extras = None # type: Optional[Tuple[STRING_TYPE, ...]] if req.extras: @@ -1097,10 +1180,14 @@ class NamedRequirement(object): @classmethod def from_pipfile(cls, name, pipfile): # type: (S, Dict[S, Union[S, bool, Union[List[S], Tuple[S, ...], Set[S]]]]) -> NamedRequirement - creation_args = {} # type: Dict[STRING_TYPE, Union[Optional[STRING_TYPE], Optional[List[STRING_TYPE]]]] + creation_args = ( + {} + ) # type: Dict[STRING_TYPE, Union[Optional[STRING_TYPE], Optional[List[STRING_TYPE]]]] if hasattr(pipfile, "keys"): attr_fields = [field.name for field in attr.fields(cls)] - creation_args = {k: v for k, v in pipfile.items() if k in attr_fields} # type: ignore + creation_args = { + k: v for k, v in pipfile.items() if k in attr_fields + } # type: ignore creation_args["name"] = name version = get_version(pipfile) # type: Optional[STRING_TYPE] extras = creation_args.get("extras", None) @@ -1148,14 +1235,18 @@ class FileRequirement(object): #: Whether the package is editable editable = attr.ib(default=False, cmp=True) # type: bool #: Extras if applicable - extras = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Tuple[STRING_TYPE, ...] + extras = attr.ib( + default=attr.Factory(tuple), cmp=True + ) # type: Tuple[STRING_TYPE, ...] _uri_scheme = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: URI of the package uri = attr.ib(cmp=True) # type: Optional[STRING_TYPE] #: Link object representing the package to clone link = attr.ib(cmp=True) # type: Optional[Link] #: PyProject Requirements - pyproject_requires = attr.ib(factory=tuple, cmp=True) # type: Optional[Tuple[STRING_TYPE, ...]] + pyproject_requires = attr.ib( + factory=tuple, cmp=True + ) # type: Optional[Tuple[STRING_TYPE, ...]] #: PyProject Build System pyproject_backend = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: PyProject Path @@ -1266,7 +1357,9 @@ class FileRequirement(object): # Re-attach VCS prefix to build a Link. link = create_link( - urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme)) # type: ignore + urllib_parse.urlunsplit( + parsed_url._replace(scheme=original_scheme) + ) # type: ignore ) return LinkInfo(vcs_type, prefer, relpath, path, uri, link) @@ -1313,7 +1406,11 @@ class FileRequirement(object): self._parsed_line is not None and self._parsed_line.requirement is not None ): self.req = self._parsed_line.requirement - if self._parsed_line and self._parsed_line.ireq and not self._parsed_line.ireq.req: + if ( + self._parsed_line + and self._parsed_line.ireq + and not self._parsed_line.ireq.req + ): if self.req is not None and self._parsed_line._ireq is not None: self._parsed_line._ireq.req = self.req @@ -1321,9 +1418,13 @@ class FileRequirement(object): def setup_info(self): # type: () -> Optional[SetupInfo] from .setup_info import SetupInfo + if self._setup_info is None and self.parsed_line: if self.parsed_line and self._parsed_line and self.parsed_line.setup_info: - if self._parsed_line._setup_info and not self._parsed_line._setup_info.name: + if ( + self._parsed_line._setup_info + and not self._parsed_line._setup_info.name + ): self._parsed_line._setup_info.get_info() self._setup_info = self.parsed_line._setup_info elif self.parsed_line and ( @@ -1350,7 +1451,11 @@ class FileRequirement(object): if self.path and not self.uri: self._uri_scheme = "path" return pip_shims.shims.path_to_url(os.path.abspath(self.path)) - elif getattr(self, "req", None) and self.req is not None and getattr(self.req, "url"): + elif ( + getattr(self, "req", None) + and self.req is not None + and getattr(self.req, "url") + ): return self.req.url elif self.link is not None: return self.link.url_without_fragment @@ -1368,18 +1473,25 @@ class FileRequirement(object): if loc: hashed_loc = hashlib.sha256(loc.encode("utf-8")).hexdigest() hashed_name = hashed_loc[-7:] - if getattr(self, "req", None) and self.req is not None and getattr(self.req, "name") and self.req.name is not None: + if ( + getattr(self, "req", None) + and self.req is not None + and getattr(self.req, "name") + and self.req.name is not None + ): if self.is_direct_url and self.req.name != hashed_name: return self.req.name if self.link and self.link.egg_fragment and self.link.egg_fragment != hashed_name: return self.link.egg_fragment elif self.link and self.link.is_wheel: from pip_shims import Wheel + self._has_hashed_name = False return Wheel(self.link.filename).name - elif self.link and ((self.link.scheme == "file" or self.editable) or ( - self.path and self.setup_path and os.path.isfile(str(self.setup_path)) - )): + elif self.link and ( + (self.link.scheme == "file" or self.editable) + or (self.path and self.setup_path and os.path.isfile(str(self.setup_path))) + ): _ireq = None # type: Optional[InstallRequirement] target_path = "" # type: STRING_TYPE if self.setup_py_dir: @@ -1401,6 +1513,7 @@ class FileRequirement(object): if self.extras and _ireq and not _ireq.extras: _ireq.extras = set(self.extras) from .setup_info import SetupInfo + subdir = getattr(self, "subdirectory", None) if self.setup_info is not None: setupinfo = self.setup_info @@ -1447,7 +1560,9 @@ class FileRequirement(object): self.name = self._parsed_line.name else: raise ValueError( - "Failed to generate a requirement: missing name for {0!r}".format(self) + "Failed to generate a requirement: missing name for {0!r}".format( + self + ) ) if self._parsed_line: try: @@ -1501,8 +1616,10 @@ class FileRequirement(object): if uri is None: if getattr(self, "path", None) and self.path is not None: uri = pip_shims.shims.path_to_url(os.path.abspath(self.path)) - elif getattr(self, "req", None) and self.req is not None and ( - getattr(self.req, "url") and self.req.url is not None + elif ( + getattr(self, "req", None) + and self.req is not None + and (getattr(self.req, "url") and self.req.url is not None) ): uri = self.req.url if uri and is_file_url(uri): @@ -1563,7 +1680,9 @@ class FileRequirement(object): if relpath and not path: path = relpath if not path and uri and link is not None and link.scheme == "file": - path = os.path.abspath(pip_shims.shims.url_to_path(unquote(uri))) # type: ignore + path = os.path.abspath( + pip_shims.shims.url_to_path(unquote(uri)) + ) # type: ignore try: path = get_converted_relative_path(path) except ValueError: # Vistir raises a ValueError if it can't make a relpath @@ -1573,7 +1692,9 @@ class FileRequirement(object): if not uri_scheme: uri_scheme = "path" if path else "file" if path and not uri: - uri = unquote(pip_shims.shims.path_to_url(os.path.abspath(path))) # type: ignore + uri = unquote( + pip_shims.shims.path_to_url(os.path.abspath(path)) + ) # type: ignore if not link and uri: link = cls.get_link_from_line(uri).link if not uri and link: @@ -1608,7 +1729,7 @@ class FileRequirement(object): "pyproject_requires": pyproject_tuple, "pyproject_backend": pyproject_backend, "path": path or relpath, - "parsed_line": parsed_line + "parsed_line": parsed_line, } if vcs_type: creation_kwargs["vcs"] = vcs_type @@ -1628,23 +1749,37 @@ class FileRequirement(object): if name: _line = "{0}#egg={1}".format(_line, name) if _line and extras and extras_to_string(extras) not in _line: - _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) # type: ignore + _line = "{0}[{1}]".format( + _line, ",".join(sorted(set(extras))) + ) # type: ignore elif isinstance(uri, six.string_types): _line = unquote(uri) elif line: _line = unquote(line) if editable: - if _line and extras and extras_to_string(extras) not in _line and ( - (link and link.scheme == "file") or (uri and uri.startswith("file")) - or (not uri and not link) + if ( + _line + and extras + and extras_to_string(extras) not in _line + and ( + (link and link.scheme == "file") + or (uri and uri.startswith("file")) + or (not uri and not link) + ) ): - _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) # type: ignore + _line = "{0}[{1}]".format( + _line, ",".join(sorted(set(extras))) + ) # type: ignore if ireq is None: - ireq = pip_shims.shims.install_req_from_editable(_line) # type: ignore + ireq = pip_shims.shims.install_req_from_editable( + _line + ) # type: ignore else: _line = path if (uri_scheme and uri_scheme == "path") else _line if _line and extras and extras_to_string(extras) not in _line: - _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) # type: ignore + _line = "{0}[{1}]".format( + _line, ",".join(sorted(set(extras))) + ) # type: ignore if ireq is None: ireq = pip_shims.shims.install_req_from_line(_line) # type: ignore if editable: @@ -1851,8 +1986,15 @@ class FileRequirement(object): def pipfile_part(self): # type: () -> Dict[AnyStr, Dict[AnyStr, Any]] excludes = [ - "_base_line", "_has_hashed_name", "setup_path", "pyproject_path", "_uri_scheme", - "pyproject_requires", "pyproject_backend", "_setup_info", "_parsed_line" + "_base_line", + "_has_hashed_name", + "setup_path", + "pyproject_path", + "_uri_scheme", + "pyproject_requires", + "pyproject_backend", + "_setup_info", + "_parsed_line", ] filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa pipfile_dict = attr.asdict(self, filter=filter_func).copy() # type: Dict @@ -1873,11 +2015,7 @@ class FileRequirement(object): key_match = next(iter(k for k in collision_order if k in pipfile_dict.keys())) if self._uri_scheme: dict_key = self._uri_scheme - target_key = ( - dict_key - if dict_key in pipfile_dict - else key_match - ) + target_key = dict_key if dict_key in pipfile_dict else key_match if target_key is not None: winning_value = pipfile_dict.pop(target_key) collisions = [k for k in collision_keys if k in pipfile_dict] @@ -1913,9 +2051,13 @@ class VCSRequirement(FileRequirement): #: URI for the repository uri = attr.ib(default=None) # type: Optional[STRING_TYPE] #: path to the repository, if it's local - path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) # type: Optional[STRING_TYPE] + path = attr.ib( + default=None, validator=attr.validators.optional(validate_path) + ) # type: Optional[STRING_TYPE] #: vcs type, i.e. git/hg/svn - vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) # type: Optional[STRING_TYPE] + vcs = attr.ib( + validator=attr.validators.optional(validate_vcs), default=None + ) # type: Optional[STRING_TYPE] #: vcs reference name (branch / commit / tag) ref = attr.ib(default=None) # type: Optional[STRING_TYPE] #: Subdirectory to use for installation if applicable @@ -1991,11 +2133,15 @@ class VCSRequirement(FileRequirement): return self._parsed_line.setup_info if self._repo: from .setup_info import SetupInfo - self._setup_info = SetupInfo.from_ireq(Line(self._repo.checkout_directory).ireq) + + self._setup_info = SetupInfo.from_ireq( + Line(self._repo.checkout_directory).ireq + ) self._setup_info.get_info() return self._setup_info ireq = self.parsed_line.ireq from .setup_info import SetupInfo + self._setup_info = SetupInfo.from_ireq(ireq) return self._setup_info @@ -2033,7 +2179,11 @@ class VCSRequirement(FileRequirement): link = self.get_link_from_line(self.uri).link if link: url = link.url_without_fragment - if url and url.startswith("git+file:/") and not url.startswith("git+file:///"): + if ( + url + and url.startswith("git+file:/") + and not url.startswith("git+file:///") + ): url = url.replace("git+file:/", "git+file:///") if url: req.url = url @@ -2051,14 +2201,20 @@ class VCSRequirement(FileRequirement): req.path = self.path req.link = self.link if ( - self.link and self.link.url_without_fragment and self.uri and - self.uri != unquote(self.link.url_without_fragment) + self.link + and self.link.url_without_fragment + and self.uri + and self.uri != unquote(self.link.url_without_fragment) and "git+ssh://" in self.link.url and "git+git@" in self.uri ): req.line = self.uri url = self.link.url_without_fragment - if url and url.startswith("git+file:/") and not url.startswith("git+file:///"): + if ( + url + and url.startswith("git+file:/") + and not url.startswith("git+file:///") + ): url = url.replace("git+file:/", "git+file:///") req.url = url return req @@ -2087,9 +2243,7 @@ class VCSRequirement(FileRequirement): checkout_dir = os.path.abspath(path) return checkout_dir if src_dir is not None: - checkout_dir = os.path.join( - os.path.abspath(src_dir), self.name - ) + checkout_dir = os.path.join(os.path.abspath(src_dir), self.name) mkdir_p(src_dir) return checkout_dir return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) @@ -2113,7 +2267,9 @@ class VCSRequirement(FileRequirement): pyproject_info = None if self.subdirectory: self.setup_path = os.path.join(checkout_dir, self.subdirectory, "setup.py") - self.pyproject_path = os.path.join(checkout_dir, self.subdirectory, "pyproject.toml") + self.pyproject_path = os.path.join( + checkout_dir, self.subdirectory, "pyproject.toml" + ) pyproject_info = get_pyproject(os.path.join(checkout_dir, self.subdirectory)) else: self.setup_path = os.path.join(checkout_dir, "setup.py") @@ -2171,8 +2327,13 @@ class VCSRequirement(FileRequirement): self._parsed_line.vcsrepo = vcsrepo if self._setup_info: self._setup_info = attr.evolve( - self._setup_info, requirements=(), _extras_requirements=(), - build_requires=(), setup_requires=(), version=None, metadata=None + self._setup_info, + requirements=(), + _extras_requirements=(), + build_requires=(), + setup_requires=(), + version=None, + metadata=None, ) if self.parsed_line and self._parsed_line: self._parsed_line.vcsrepo = vcsrepo @@ -2293,7 +2454,7 @@ class VCSRequirement(FileRequirement): "line": line, "uri": uri, "uri_scheme": prefer, - "parsed_line": parsed_line + "parsed_line": parsed_line, } if relpath: creation_args["relpath"] = relpath @@ -2309,7 +2470,7 @@ class VCSRequirement(FileRequirement): uri=uri, extras=extras_tuple if extras_tuple else tuple(), base_line=line, - parsed_line=parsed_line + parsed_line=parsed_line, ) if cls_inst.req and ( cls_inst._parsed_line.ireq and not cls_inst.parsed_line.ireq.req @@ -2377,9 +2538,16 @@ class VCSRequirement(FileRequirement): def pipfile_part(self): # type: () -> Dict[S, Dict[S, Union[List[S], S, bool, RequirementType, pip_shims.shims.Link]]] excludes = [ - "_repo", "_base_line", "setup_path", "_has_hashed_name", "pyproject_path", - "pyproject_requires", "pyproject_backend", "_setup_info", "_parsed_line", - "_uri_scheme" + "_repo", + "_base_line", + "setup_path", + "_has_hashed_name", + "pyproject_path", + "pyproject_requires", + "pyproject_backend", + "_setup_info", + "_parsed_line", + "_uri_scheme", ] filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa pipfile_dict = attr.asdict(self, filter=filter_func).copy() @@ -2400,17 +2568,27 @@ class VCSRequirement(FileRequirement): @attr.s(cmp=True, hash=True) class Requirement(object): _name = attr.ib(cmp=True) # type: STRING_TYPE - vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs), cmp=True) # type: Optional[STRING_TYPE] - req = attr.ib(default=None, cmp=True) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] + vcs = attr.ib( + default=None, validator=attr.validators.optional(validate_vcs), cmp=True + ) # type: Optional[STRING_TYPE] + req = attr.ib( + default=None, cmp=True + ) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] markers = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] - _specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers), cmp=True) # type: Optional[STRING_TYPE] + _specifiers = attr.ib( + validator=attr.validators.optional(validate_specifiers), cmp=True + ) # type: Optional[STRING_TYPE] index = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] editable = attr.ib(default=None, cmp=True) # type: Optional[bool] - hashes = attr.ib(factory=frozenset, converter=frozenset, cmp=True) # type: FrozenSet[STRING_TYPE] + hashes = attr.ib( + factory=frozenset, converter=frozenset, cmp=True + ) # type: FrozenSet[STRING_TYPE] extras = attr.ib(factory=tuple, cmp=True) # type: Tuple[STRING_TYPE, ...] abstract_dep = attr.ib(default=None, cmp=False) # type: Optional[AbstractDependency] _line_instance = attr.ib(default=None, cmp=False) # type: Optional[Line] - _ireq = attr.ib(default=None, cmp=False) # type: Optional[pip_shims.InstallRequirement] + _ireq = attr.ib( + default=None, cmp=False + ) # type: Optional[pip_shims.InstallRequirement] def __hash__(self): return hash(self.as_line()) @@ -2511,6 +2689,7 @@ class Requirement(object): def update_name_from_path(self, path): from .setup_info import get_metadata + metadata = get_metadata(path) name = self.name if metadata is not None: @@ -2572,15 +2751,21 @@ class Requirement(object): self._specifiers = specs return specs if not self._specifiers and ( - self.req is not None and - isinstance(self.req, NamedRequirement) and - self.req.version + self.req is not None + and isinstance(self.req, NamedRequirement) + and self.req.version ): self._specifiers = self.req.version - elif not self.editable and self.req and ( - not isinstance(self.req, NamedRequirement) and self.req.setup_info + elif ( + not self.editable + and self.req + and (not isinstance(self.req, NamedRequirement) and self.req.setup_info) ): - if self.line_instance and self.line_instance.setup_info and self.line_instance.setup_info.version: + if ( + self.line_instance + and self.line_instance.setup_info + and self.line_instance.setup_info.version + ): self._specifiers = "=={0}".format(self.req.setup_info.version) elif not self._specifiers: if self.req and self.req.parsed_line and self.req.parsed_line.specifiers: @@ -2643,9 +2828,10 @@ class Requirement(object): @property def is_wheel(self): # type: () -> bool - if self.req and not isinstance(self.req, NamedRequirement) and ( - self.req.link is not None and - self.req.link.is_wheel + if ( + self.req + and not isinstance(self.req, NamedRequirement) + and (self.req.link is not None and self.req.link.is_wheel) ): return True return False @@ -2665,8 +2851,12 @@ class Requirement(object): if isinstance(line, pip_shims.shims.InstallRequirement): line = format_requirement(line) parsed_line = Line(line) - r = None # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] - if ((parsed_line.is_file and parsed_line.is_installable) or parsed_line.is_url) and not parsed_line.is_vcs: + r = ( + None + ) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] + if ( + (parsed_line.is_file and parsed_line.is_installable) or parsed_line.is_url + ) and not parsed_line.is_vcs: r = file_req_from_parsed_line(parsed_line) elif parsed_line.is_vcs: r = vcs_req_from_parsed_line(parsed_line) @@ -2688,7 +2878,7 @@ class Requirement(object): "req": r, "markers": parsed_line.markers, "editable": parsed_line.editable, - "line_instance": parsed_line + "line_instance": parsed_line, } if parsed_line.extras: extras = () # type: Tuple[STRING_TYPE, ...] @@ -2697,7 +2887,9 @@ class Requirement(object): if r is not None: r.extras = extras elif r is not None and r.extras is not None: - args["extras"] = tuple(sorted(dedup([extra.lower() for extra in r.extras]))) # type: ignore + args["extras"] = tuple( + sorted(dedup([extra.lower() for extra in r.extras])) + ) # type: ignore if r.req is not None: r.req.extras = args["extras"] if parsed_line.hashes: @@ -2743,7 +2935,9 @@ class Requirement(object): if r.req.specifier: r.req.specifier = SpecifierSet(_pipfile["version"]) r.req.extras = ( - tuple(sorted(dedup([extra.lower() for extra in extras]))) if extras else () + tuple(sorted(dedup([extra.lower() for extra in extras]))) + if extras + else () ) args = { "name": r.name, @@ -2851,8 +3045,10 @@ class Requirement(object): @property def is_direct_url(self): - return self.is_file_or_url and self.req.is_direct_url or ( - self.line_instance.is_direct_url or self.req.parsed_line.is_direct_url + return ( + self.is_file_or_url + and self.req.is_direct_url + or (self.line_instance.is_direct_url or self.req.parsed_line.is_direct_url) ) def as_pipfile(self): @@ -2876,13 +3072,15 @@ class Requirement(object): name_carriers = (self.req, self, self.line_instance, self.req.parsed_line) name_options = [ getattr(carrier, "name", None) - for carrier in name_carriers if carrier is not None + for carrier in name_carriers + if carrier is not None ] req_name = next(iter(n for n in name_options if n is not None), None) self.req.name = req_name req_name, dict_from_subreq = self.req.pipfile_part.popitem() base_dict = { - k: v for k, v in dict_from_subreq.items() + k: v + for k, v in dict_from_subreq.items() if k not in ["req", "link", "_setup_info"] } base_dict.update(req_dict) @@ -2911,9 +3109,7 @@ class Requirement(object): return self.line_instance.ireq elif getattr(self.req, "_parsed_line", None) and self.req._parsed_line.ireq: return self.req._parsed_line.ireq - kwargs = { - "include_hashes": False, - } + kwargs = {"include_hashes": False} if (self.is_file_or_url and self.req.is_local) or self.is_vcs: kwargs["include_markers"] = False ireq_line = self.as_line(**kwargs) @@ -2986,9 +3182,7 @@ class Requirement(object): else: ireq = sorted(self.find_all_matches(), key=lambda k: k.version) deps = get_dependencies(ireq.pop(), sources=sources) - return get_abstract_dependencies( - deps, sources=sources, parent=self.abstract_dep - ) + return get_abstract_dependencies(deps, sources=sources, parent=self.abstract_dep) def find_all_matches(self, sources=None, finder=None): # type: (Optional[List[Dict[S, Union[S, bool]]]], Optional[PackageFinder]) -> List[InstallationCandidate] @@ -3016,8 +3210,10 @@ class Requirement(object): info_dict = self.line_instance.setup_info.as_dict() else: from .setup_info import SetupInfo + if not finder: from .dependencies import get_finder + finder = get_finder(sources=sources) info = SetupInfo.from_requirement(self, finder=finder) if info is None: @@ -3062,9 +3258,11 @@ def file_req_from_parsed_line(parsed_line): "uri": parsed_line.uri, "pyproject_requires": pyproject_requires, "pyproject_backend": parsed_line.pyproject_backend, - "pyproject_path": Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None, + "pyproject_path": Path(parsed_line.pyproject_toml) + if parsed_line.pyproject_toml + else None, "parsed_line": parsed_line, - "req": parsed_line.requirement + "req": parsed_line.requirement, } if parsed_line.name is not None: req_dict["name"] = parsed_line.name @@ -3077,14 +3275,16 @@ def vcs_req_from_parsed_line(parsed_line): if parsed_line.editable: line = "-e {0}".format(line) if parsed_line.url is not None: - link = create_link(build_vcs_uri( - vcs=parsed_line.vcs, - uri=parsed_line.url, - name=parsed_line.name, - ref=parsed_line.ref, - subdirectory=parsed_line.subdirectory, - extras=list(parsed_line.extras) - )) + link = create_link( + build_vcs_uri( + vcs=parsed_line.vcs, + uri=parsed_line.url, + name=parsed_line.name, + ref=parsed_line.ref, + subdirectory=parsed_line.subdirectory, + extras=list(parsed_line.extras), + ) + ) else: link = parsed_line.link pyproject_requires = () # type: Optional[Tuple[STRING_TYPE, ...]] @@ -3103,10 +3303,12 @@ def vcs_req_from_parsed_line(parsed_line): "uri": parsed_line.uri, "pyproject_requires": pyproject_requires, "pyproject_backend": parsed_line.pyproject_backend, - "pyproject_path": Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None, + "pyproject_path": Path(parsed_line.pyproject_toml) + if parsed_line.pyproject_toml + else None, "parsed_line": parsed_line, "req": parsed_line.requirement, - "base_line": line + "base_line": line, } if parsed_line.name: vcs_dict["name"] = parsed_line.name @@ -3122,6 +3324,6 @@ def named_req_from_parsed_line(parsed_line): req=parsed_line.requirement, extras=parsed_line.extras, editable=parsed_line.editable, - parsed_line=parsed_line + parsed_line=parsed_line, ) return NamedRequirement.from_line(parsed_line.line) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index c8f9e93f..2aecfa43 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -19,22 +19,21 @@ from distlib.wheel import Wheel from packaging.markers import Marker from six.moves import configparser from six.moves.urllib.parse import unquote, urlparse, urlunparse - from vistir.compat import Iterable, Path, lru_cache from vistir.contextmanagers import cd, temp_path from vistir.misc import run from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p, rmtree -from ..environment import MYPY_RUNNING -from ..exceptions import RequirementError from .utils import ( + get_default_pyproject_backend, get_name_variants, get_pyproject, init_requirement, split_vcs_method_from_uri, strip_extras_markers_from_requirement, - get_default_pyproject_backend ) +from ..environment import MYPY_RUNNING +from ..exceptions import RequirementError try: from setuptools.dist import distutils @@ -49,15 +48,32 @@ except ImportError: if MYPY_RUNNING: - from typing import Any, Dict, List, Generator, Optional, Union, Tuple, TypeVar, Text, Set, AnyStr + from typing import ( + Any, + Dict, + List, + Generator, + Optional, + Union, + Tuple, + TypeVar, + Text, + Set, + AnyStr, + ) from pip_shims.shims import InstallRequirement, PackageFinder from pkg_resources import ( - PathMetadata, DistInfoDistribution, Requirement as PkgResourcesRequirement + PathMetadata, + DistInfoDistribution, + Requirement as PkgResourcesRequirement, ) from packaging.requirements import Requirement as PackagingRequirement + TRequirement = TypeVar("TRequirement") - RequirementType = TypeVar('RequirementType', covariant=True, bound=PackagingRequirement) - MarkerType = TypeVar('MarkerType', covariant=True, bound=Marker) + RequirementType = TypeVar( + "RequirementType", covariant=True, bound=PackagingRequirement + ) + MarkerType = TypeVar("MarkerType", covariant=True, bound=Marker) STRING_TYPE = Union[str, bytes, Text] S = TypeVar("S", bytes, str, Text) @@ -77,16 +93,37 @@ def pep517_subprocess_runner(cmd, cwd=None, extra_environ=None): if extra_environ: env.update(extra_environ) - run(cmd, cwd=cwd, env=env, block=True, combine_stderr=True, return_object=False, - write_to_stdout=False, nospin=True) + run( + cmd, + cwd=cwd, + env=env, + block=True, + combine_stderr=True, + return_object=False, + write_to_stdout=False, + nospin=True, + ) class BuildEnv(pep517.envbuild.BuildEnvironment): def pip_install(self, reqs): - cmd = [sys.executable, '-m', 'pip', 'install', '--ignore-installed', '--prefix', - self.path] + list(reqs) - run(cmd, block=True, combine_stderr=True, return_object=False, - write_to_stdout=False, nospin=True) + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--ignore-installed", + "--prefix", + self.path, + ] + list(reqs) + run( + cmd, + block=True, + combine_stderr=True, + return_object=False, + write_to_stdout=False, + nospin=True, + ) class HookCaller(pep517.wrappers.Pep517HookCaller): @@ -156,6 +193,7 @@ def _get_src_dir(root): def ensure_reqs(reqs): # type: (List[Union[S, PkgResourcesRequirement]]) -> List[PkgResourcesRequirement] import pkg_resources + if not isinstance(reqs, Iterable): raise TypeError("Expecting an Iterable, got %r" % reqs) new_reqs = [] @@ -169,7 +207,9 @@ def ensure_reqs(reqs): return new_reqs -def _prepare_wheel_building_kwargs(ireq=None, src_root=None, src_dir=None, editable=False): +def _prepare_wheel_building_kwargs( + ireq=None, src_root=None, src_dir=None, editable=False +): # type: (Optional[InstallRequirement], Optional[AnyStr], Optional[AnyStr], bool) -> Dict[AnyStr, AnyStr] download_dir = os.path.join(CACHE_DIR, "pkgs") # type: STRING_TYPE mkdir_p(download_dir) @@ -214,14 +254,17 @@ def iter_metadata(path, pkg_name=None, metadata_type="egg-info"): elif not entry.name.endswith(metadata_type): non_matching_dirs.append(entry) for entry in non_matching_dirs: - for dir_entry in iter_metadata(entry.path, pkg_name=pkg_name, metadata_type=metadata_type): + for dir_entry in iter_metadata( + entry.path, pkg_name=pkg_name, metadata_type=metadata_type + ): yield dir_entry def find_egginfo(target, pkg_name=None): # type: (AnyStr, Optional[AnyStr]) -> Generator egg_dirs = ( - egg_dir for egg_dir in iter_metadata(target, pkg_name=pkg_name) + egg_dir + for egg_dir in iter_metadata(target, pkg_name=pkg_name) if egg_dir is not None ) if pkg_name: @@ -234,7 +277,10 @@ def find_egginfo(target, pkg_name=None): def find_distinfo(target, pkg_name=None): # type: (AnyStr, Optional[AnyStr]) -> Generator dist_dirs = ( - dist_dir for dist_dir in iter_metadata(target, pkg_name=pkg_name, metadata_type="dist-info") + dist_dir + for dist_dir in iter_metadata( + target, pkg_name=pkg_name, metadata_type="dist-info" + ) if dist_dir is not None ) if pkg_name: @@ -260,6 +306,7 @@ def get_metadata(path, pkg_name=None, metadata_type=None): base_dir = None if matched_dir is not None: import pkg_resources + metadata_dir = os.path.abspath(matched_dir.path) base_dir = os.path.dirname(metadata_dir) dist = None @@ -320,12 +367,7 @@ def get_metadata_from_wheel(wheel_path): extras[extra].append(parsed_req) else: requires.append(parsed_req) - return { - "name": name, - "version": version, - "requires": requires, - "extras": extras - } + return {"name": name, "version": version, "requires": requires, "extras": extras} def get_metadata_from_dist(dist): @@ -362,14 +404,16 @@ def get_metadata_from_dist(dist): "name": dist.project_name, "version": dist.version, "requires": requires, - "extras": extras + "extras": extras, } @attr.s(slots=True, frozen=True) class BaseRequirement(object): name = attr.ib(default="", cmp=True) # type: STRING_TYPE - requirement = attr.ib(default=None, cmp=True) # type: Optional[PkgResourcesRequirement] + requirement = attr.ib( + default=None, cmp=True + ) # type: Optional[PkgResourcesRequirement] def __str__(self): # type: () -> S @@ -413,12 +457,16 @@ class Extra(object): def __str__(self): # type: () -> S - return "{0}: {{{1}}}".format(self.section, ", ".join([r.name for r in self.requirements])) + return "{0}: {{{1}}}".format( + self.section, ", ".join([r.name for r in self.requirements]) + ) def add(self, req): # type: (BaseRequirement) -> None if req not in self.requirements: - return attr.evolve(self, requirements=frozenset(set(self.requirements).add(req))) + return attr.evolve( + self, requirements=frozenset(set(self.requirements).add(req)) + ) return self def as_dict(self): @@ -435,12 +483,16 @@ class SetupInfo(object): build_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) build_backend = attr.ib(cmp=True) # type: STRING_TYPE setup_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) - python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None, cmp=True) + python_requires = attr.ib( + type=packaging.specifiers.SpecifierSet, default=None, cmp=True + ) _extras_requirements = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) setup_cfg = attr.ib(type=Path, default=None, cmp=True, hash=False) setup_py = attr.ib(type=Path, default=None, cmp=True, hash=False) pyproject = attr.ib(type=Path, default=None, cmp=True, hash=False) - ireq = attr.ib(default=None, cmp=True, hash=False) # type: Optional[InstallRequirement] + ireq = attr.ib( + default=None, cmp=True, hash=False + ) # type: Optional[InstallRequirement] extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict, cmp=False, hash=False) metadata = attr.ib(default=None) # type: Optional[Tuple[STRING_TYPE]] @@ -452,7 +504,9 @@ class SetupInfo(object): @property def requires(self): # type: () -> Dict[S, RequirementType] - return {req.name: req.requirement for req in self._requirements} + return { + req.name: req.requirement for req in self._requirements + } @property def extras(self): @@ -489,30 +543,34 @@ class SetupInfo(object): results["version"] = parser.get("metadata", "version") install_requires = set() # type: Set[BaseRequirement] if parser.has_option("options", "install_requires"): - install_requires = set([ - BaseRequirement.from_string(dep) - for dep in parser.get("options", "install_requires").split("\n") - if dep - ]) + install_requires = set( + [ + BaseRequirement.from_string(dep) + for dep in parser.get("options", "install_requires").split("\n") + if dep + ] + ) results["install_requires"] = install_requires if parser.has_option("options", "python_requires"): results["python_requires"] = parser.get("options", "python_requires") if parser.has_option("options", "build_requires"): results["build_requires"] = parser.get("options", "build_requires") - extras_require = () if "options.extras_require" in parser.sections(): - extras_require = tuple([ - (section, tuple([ - BaseRequirement.from_string(dep) - for dep in parser.get( - "options.extras_require", section - ).split("\n") - if dep - ])) - for section in parser.options("options.extras_require") - if section not in ["options", "metadata"] - ]) - results["extras_require"] = extras_require + extras_require_section = parser.options("options.extras_require") + extras = [] + for section in extras_require_section: + if section in ["options", "metadata"]: + continue + section_contents = parser.get("options.extras_require", section) + section_list = section_contents.split("\n") + section_extras = [] + for extra_name in section_list: + if not extra_name or extra_name.startswith("#"): + continue + section_extras.append(BaseRequirement.from_string(extra_name)) + if section_extras: + extras.append(tuple([section, tuple(section_extras)])) + results["extras_require"] = tuple(extras) return results @property @@ -545,20 +603,26 @@ class SetupInfo(object): self.version = parsed.get("version") build_requires = parsed.get("build_requires", []) if self.build_requires: - self.build_requires = tuple(set(self.build_requires) | set(build_requires)) + self.build_requires = tuple( + set(self.build_requires) | set(build_requires) + ) self._requirements = frozenset( set(self._requirements) | set(parsed["install_requires"]) ) if self.python_requires is None: self.python_requires = parsed.get("python_requires") if not self._extras_requirements: - self._extras_requirements = (parsed["extras_require"]) + self._extras_requirements = parsed["extras_require"] else: - self._extras_requirements = self._extras_requirements + parsed["extras_require"] + self._extras_requirements = ( + self._extras_requirements + parsed["extras_require"] + ) if self.ireq is not None and self.ireq.extras: for extra in self.ireq.extras: if extra in self.extras: - extras_tuple = tuple([BaseRequirement.from_req(req) for req in self.extras[extra]]) + extras_tuple = tuple( + [BaseRequirement.from_req(req) for req in self.extras[extra]] + ) self._extras_requirements += ((extra, extras_tuple),) self._requirements = frozenset( set(self._requirements) | set(list(extras_tuple)) @@ -585,16 +649,22 @@ class SetupInfo(object): _setup_stop_after = "run" sys.argv[0] = script_name sys.argv[1:] = args - with open(script_name, 'rb') as f: + with open(script_name, "rb") as f: if sys.version_info < (3, 5): exec(f.read(), g, local_dict) else: exec(f.read(), g) # We couldn't import everything needed to run setup except NameError: - python = os.environ.get('PIP_PYTHON_PATH', sys.executable) - out, _ = run([python, "setup.py"] + args, cwd=target_cwd, block=True, - combine_stderr=False, return_object=False, nospin=True) + python = os.environ.get("PIP_PYTHON_PATH", sys.executable) + out, _ = run( + [python, "setup.py"] + args, + cwd=target_cwd, + block=True, + combine_stderr=False, + return_object=False, + nospin=True, + ) finally: _setup_stop_after = None sys.argv = save_argv @@ -622,15 +692,13 @@ class SetupInfo(object): if not install_requires: install_requires = dist.install_requires if install_requires and not self.requires: - requirements = set([ - BaseRequirement.from_req(req) for req in install_requires - ]) + requirements = set( + [BaseRequirement.from_req(req) for req in install_requires] + ) if getattr(self.ireq, "extras", None): for extra in self.ireq.extras: requirements |= set(list(self.extras.get(extra, []))) - self._requirements = frozenset( - set(self._requirements) | requirements - ) + self._requirements = frozenset(set(self._requirements) | requirements) if dist.setup_requires and not self.setup_requires: self.setup_requires = tuple(dist.setup_requires) if not self.version: @@ -647,15 +715,20 @@ class SetupInfo(object): # type: () -> S if not self.pyproject.exists(): build_requires = ", ".join(['"{0}"'.format(r) for r in self.build_requires]) - self.pyproject.write_text(u""" + self.pyproject.write_text( + u""" [build-system] requires = [{0}] build-backend = "{1}" - """.format(build_requires, self.build_backend).strip()) + """.format( + build_requires, self.build_backend + ).strip() + ) return build_pep517( - self.base_dir, self.extra_kwargs["build_dir"], + self.base_dir, + self.extra_kwargs["build_dir"], config_settings=self.pep517_config, - dist_type="wheel" + dist_type="wheel", ) # noinspection PyPackageRequirements @@ -663,15 +736,20 @@ build-backend = "{1}" # type: () -> S if not self.pyproject.exists(): build_requires = ", ".join(['"{0}"'.format(r) for r in self.build_requires]) - self.pyproject.write_text(u""" + self.pyproject.write_text( + u""" [build-system] requires = [{0}] build-backend = "{1}" - """.format(build_requires, self.build_backend).strip()) + """.format( + build_requires, self.build_backend + ).strip() + ) return build_pep517( - self.base_dir, self.extra_kwargs["build_dir"], + self.base_dir, + self.extra_kwargs["build_dir"], config_settings=self.pep517_config, - dist_type="sdist" + dist_type="sdist", ) def build(self): @@ -719,12 +797,17 @@ build-backend = "{1}" # if self.setup_py is not None and self.setup_py.exists(): metadata_dirs = [] if any([fn is not None and fn.exists() for fn in package_indicators]): - metadata_dirs = [self.extra_kwargs["build_dir"], self.egg_base, self.extra_kwargs["src_dir"]] + metadata_dirs = [ + self.extra_kwargs["build_dir"], + self.egg_base, + self.extra_kwargs["src_dir"], + ] if metadata_dir is not None: metadata_dirs = [metadata_dir] + metadata_dirs metadata = [ get_metadata(d, pkg_name=self.name, metadata_type=metadata_type) - for d in metadata_dirs if os.path.exists(d) + for d in metadata_dirs + if os.path.exists(d) ] metadata = next(iter(d for d in metadata if d), None) if metadata is not None: @@ -749,20 +832,20 @@ build-backend = "{1}" if not self.version: self.version = metadata.get("version", self.version) self._requirements = frozenset( - set(self._requirements) | set([ - BaseRequirement.from_req(req) - for req in metadata.get("requires", []) - ]) + set(self._requirements) + | set([BaseRequirement.from_req(req) for req in metadata.get("requires", [])]) ) if getattr(self.ireq, "extras", None): for extra in self.ireq.extras: extras = metadata.get("extras", {}).get(extra, []) if extras: - extras_tuple = tuple([ - BaseRequirement.from_req(req) - for req in ensure_reqs(tuple(extras)) - if req is not None - ]) + extras_tuple = tuple( + [ + BaseRequirement.from_req(req) + for req in ensure_reqs(tuple(extras)) + if req is not None + ] + ) self._extras_requirements += ((extra, extras_tuple),) self._requirements = frozenset( set(self._requirements) | set(extras_tuple) @@ -839,6 +922,7 @@ build-backend = "{1}" def from_ireq(cls, ireq, subdir=None, finder=None): # type: (InstallRequirement, Optional[AnyStr], Optional[PackageFinder]) -> Optional[SetupInfo] import pip_shims.shims + if not ireq.link: return if ireq.link.is_wheel: @@ -875,8 +959,7 @@ build-backend = "{1}" download_dir = kwargs["download_dir"] elif path is not None and os.path.isdir(path): raise RequirementError( - "The file URL points to a directory not installable: {}" - .format(ireq.link) + "The file URL points to a directory not installable: {}".format(ireq.link) ) ireq.build_location(kwargs["build_dir"]) src_dir = ireq.ensure_has_source_dir(kwargs["src_dir"]) @@ -892,9 +975,7 @@ build-backend = "{1}" hashes=ireq.hashes(False), progress_bar="off", ) - created = cls.create( - src_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs - ) + created = cls.create(src_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs) return created @classmethod From de4d659eac72f64b82d6e4bccefcbc524aba99d7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 3 Mar 2019 00:22:40 -0500 Subject: [PATCH 09/21] requirementslib bugfix Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/setup_info.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 2aecfa43..0f7bc177 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -504,9 +504,7 @@ class SetupInfo(object): @property def requires(self): # type: () -> Dict[S, RequirementType] - return { - req.name: req.requirement for req in self._requirements - } + return {req.name: req.requirement for req in self._requirements} @property def extras(self): @@ -555,9 +553,9 @@ class SetupInfo(object): results["python_requires"] = parser.get("options", "python_requires") if parser.has_option("options", "build_requires"): results["build_requires"] = parser.get("options", "build_requires") + extras = [] if "options.extras_require" in parser.sections(): extras_require_section = parser.options("options.extras_require") - extras = [] for section in extras_require_section: if section in ["options", "metadata"]: continue From 374086e31872fa93051d3bbf9d5010d8e7e97c92 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 4 Mar 2019 01:07:18 -0500 Subject: [PATCH 10/21] Update requirementslib Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/__init__.py | 2 +- .../requirementslib/models/requirements.py | 173 +++++++++--------- pipenv/vendor/requirementslib/models/utils.py | 128 ++++++++----- pipenv/vendor/requirementslib/utils.py | 108 +++++++---- 4 files changed, 242 insertions(+), 169 deletions(-) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 8246f036..54e71e8d 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,6 +1,6 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function -__version__ = '1.4.1' +__version__ = '1.4.2' import logging import warnings diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index b60f8ca5..5fb219ac 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -41,6 +41,7 @@ from vistir.path import ( ) from .setup_info import SetupInfo, _prepare_wheel_building_kwargs +from .url import URI from .utils import ( DIRECT_URL_RE, HASH_STRING, @@ -87,6 +88,7 @@ if MYPY_RUNNING: Union, Any, Tuple, + Sequence, Set, AnyStr, Text, @@ -114,6 +116,9 @@ if MYPY_RUNNING: BASE_TYPES = Union[bool, STRING_TYPE, Tuple[STRING_TYPE, ...]] CUSTOM_TYPES = Union[VCSRepository, RequirementType, SetupInfo, "Line"] CREATION_ARG_TYPES = Union[BASE_TYPES, Link, CUSTOM_TYPES] + PIPFILE_ENTRY_TYPE = Union[STRING_TYPE, bool, Tuple[STRING_TYPE], List[STRING_TYPE]] + PIPFILE_TYPE = Union[STRING_TYPE, Dict[STRING_TYPE, PIPFILE_ENTRY_TYPE]] + TPIPFILE = Dict[STRING_TYPE, PIPFILE_ENTRY_TYPE] SPECIFIERS_BY_LENGTH = sorted(list(Specifier._operators.keys()), key=len, reverse=True) @@ -249,10 +254,21 @@ class Line(object): line = os.path.abspath(self.base_path) else: if DIRECT_URL_RE.match(self.line): + uri = URI.parse(self.line) + line = uri.full_url self._requirement = init_requirement(self.line) line = convert_direct_url_to_url(self.line) else: - line = self.link.url + if self.link: + line = self.link.url + else: + try: + uri = URI.parse(line) + except ValueError: + line = line + else: + line = uri.base_url + self._link = uri.as_link if self.editable: if not line: @@ -433,30 +449,35 @@ class Line(object): def pyproject_requires(self): # type: () -> Optional[Tuple[STRING_TYPE, ...]] if self._pyproject_requires is None and self.pyproject_toml is not None: - pyproject_requires, pyproject_backend = get_pyproject( - self.path - ) # type: ignore - if pyproject_requires: - self._pyproject_requires = tuple(pyproject_requires) - self._pyproject_backend = pyproject_backend + if self.path is not None: + pyproject_requires, pyproject_backend = None, None + pyproject_results = get_pyproject(self.path) # type: ignore + if pyproject_results: + pyproject_requires, pyproject_backend = pyproject_results + if pyproject_requires: + self._pyproject_requires = tuple(pyproject_requires) + self._pyproject_backend = pyproject_backend return self._pyproject_requires @property def pyproject_backend(self): # type: () -> Optional[STRING_TYPE] if self._pyproject_requires is None and self.pyproject_toml is not None: - pyproject_requires, pyproject_backend = get_pyproject( - self.path - ) # type: ignore + pyproject_requires = None # type: Optional[Sequence[STRING_TYPE]] + pyproject_backend = None # type: Optional[STRING_TYPE] + pyproject_results = get_pyproject(self.path) # type: ignore + if pyproject_results: + pyproject_requires, pyproject_backend = pyproject_results if not pyproject_backend and self.setup_cfg is not None: setup_dict = SetupInfo.get_setup_cfg(self.setup_cfg) pyproject_backend = get_default_pyproject_backend() pyproject_requires = setup_dict.get( "build_requires", ["setuptools", "wheel"] ) # type: ignore - - self._pyproject_requires = tuple(pyproject_requires) - self._pyproject_backend = pyproject_backend + if pyproject_requires: + self._pyproject_requires = tuple(pyproject_requires) + if pyproject_backend: + self._pyproject_backend = pyproject_backend return self._pyproject_backend def parse_hashes(self): @@ -480,45 +501,16 @@ class Line(object): """ extras = None - url = "" # type: STRING_TYPE if "@" in self.line or self.is_vcs or self.is_url: line = "{0}".format(self.line) - match = DIRECT_URL_RE.match(line) - if match is None: - match = URL_RE.match(line) - else: - self.is_direct_url = True - if match is not None: - match_dict = match.groupdict() - name = match_dict.get("name") - extras = match_dict.get("extras") - scheme = match_dict.get("scheme") - host = match_dict.get("host") - path = match_dict.get("path") - ref = match_dict.get("ref") - subdir = match_dict.get("subdirectory") - pathsep = match_dict.get("pathsep", "/") - if scheme is not None: - url = scheme - if host: - url = "{0}{1}".format(url, host) - if path: - url = "{0}{1}{2}".format(url, pathsep, path) - if self.is_vcs and ref: - url = "{0}@{1}".format(url, ref) - if name: - url = "{0}#egg={1}".format(url, name) - if extras: - url = "{0}{1}".format(url, extras) - elif is_file_url(url) and extras and not name and self.editable: - url = "{0}{1}{2}".format(pathsep, path, extras) - if subdir: - url = "{0}&subdirectory={1}".format(url, subdir) - elif extras and not path: - url = "{0}{1}".format(url, extras) - self.line = add_ssh_scheme_to_git_uri(url) - if name: - self._name = name + uri = URI.parse(line) + name = uri.name + if name: + self._name = name + if uri.host and uri.path and uri.scheme: + self.line = uri.to_string( + escape_password=False, direct=False, strip_ssh=uri.is_implicit_ssh + ) else: self.line, extras = pip_shims.shims._strip_extras(self.line) else: @@ -539,6 +531,14 @@ class Line(object): """Sets ``self.name`` if given a **PEP-508** style URL""" line = self.line + try: + parsed = URI.parse(line) + line = parsed.to_string(escape_password=False, direct=False, strip_ref=True) + except ValueError: + pass + else: + self._parsed_url = parsed + return line if self.vcs is not None and self.line.startswith("{0}+".format(self.vcs)): _, _, _parseable = self.line.partition("+") parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(_parseable)) @@ -985,7 +985,31 @@ class Line(object): def parse_link(self): # type: () -> None - if self.is_file or self.is_url or self.is_vcs: + parsed_url = None # type: Optional[URI] + if not is_valid_url(self.line) and ( + self.line.startswith("./") + or (os.path.exists(self.line) or os.path.isabs(self.line)) + ): + url = pip_shims.shims.path_to_url(os.path.abspath(self.line)) + parsed_url = URI.parse(url) + elif is_valid_url(self.line) or is_vcs(self.line) or is_file_url(self.line): + parsed_url = URI.parse(self.line) + if parsed_url is not None: + line = parsed_url.to_string( + escape_password=False, direct=False, strip_ref=True, strip_ssh=False + ) + if parsed_url.is_vcs: + self.vcs, _ = parsed_url.scheme.split("+") + if parsed_url.is_file_url: + self.is_local = True + parsed_link = parsed_url.as_link + self._ref = parsed_url.ref + self.uri = parsed_url.bare_url + if parsed_url.name: + self._name = parsed_url.name + if parsed_url.extras: + self.extras = tuple(sorted(set(parsed_url.extras))) + self._link = parsed_link vcs, prefer, relpath, path, uri, link = FileRequirement.get_link_from_line( self.line ) @@ -999,32 +1023,14 @@ class Line(object): link_url = link.url_without_fragment if "@" in link_url: link_url, _ = split_ref_from_uri(link_url) - self._ref = ref - self.vcs = vcs self.preferred_scheme = prefer self.relpath = relpath self.path = path - self.uri = uri + # self.uri = uri if prefer in ("path", "relpath") or uri.startswith("file"): self.is_local = True - if link.egg_fragment: - name, extras = pip_shims.shims._strip_extras(link.egg_fragment) - self.extras = tuple(sorted(set(parse_extras(extras)))) - self._name = name - else: - # set this so we can call `self.name` without a recursion error - self._link = link - if (self.is_direct_url or vcs) and self.name is not None and vcs is not None: - self._link = create_link( - build_vcs_uri( - vcs=vcs, - uri=link_url, - ref=ref, - extras=self.extras, - name=self.name, - subdirectory=link.subdirectory_fragment, - ) - ) + if parsed_url.is_vcs or parsed_url.is_direct_url and parsed_link: + self._link = parsed_link else: self._link = link @@ -1041,7 +1047,7 @@ class Line(object): Generates a 3-tuple of the requisite *name*, *extras* and *url* to generate a :class:`~packaging.requirements.Requirement` out of. - :return: A Tuple containing an optional name, a Tuple of extras names, and an optional URL. + :return: A Tuple of an optional name, a Tuple of extras, and an optional URL. :rtype: Tuple[Optional[S], Tuple[Optional[S], ...], Optional[S]] """ @@ -1083,8 +1089,8 @@ class Line(object): when there is a folder called *alembic* in the working directory. In this case we first need to check that the given requirement is a valid - URL, VCS requirement, or installable filesystem path before deciding to treat it as - a file requirement over a named requirement. + URL, VCS requirement, or installable filesystem path before deciding to treat it + as a file requirement over a named requirement. """ line = self.line if is_file_url(line): @@ -1178,11 +1184,9 @@ class NamedRequirement(object): return cls(**creation_kwargs) @classmethod - def from_pipfile(cls, name, pipfile): - # type: (S, Dict[S, Union[S, bool, Union[List[S], Tuple[S, ...], Set[S]]]]) -> NamedRequirement - creation_args = ( - {} - ) # type: Dict[STRING_TYPE, Union[Optional[STRING_TYPE], Optional[List[STRING_TYPE]]]] + def from_pipfile(cls, name, pipfile): # type: S # type: TPIPFILE + # type: (...) -> NamedRequirement + creation_args = {} # type: TPIPFILE if hasattr(pipfile, "keys"): attr_fields = [field.name for field in attr.fields(cls)] creation_args = { @@ -1191,10 +1195,13 @@ class NamedRequirement(object): creation_args["name"] = name version = get_version(pipfile) # type: Optional[STRING_TYPE] extras = creation_args.get("extras", None) - creation_args["version"] = version + creation_args["version"] = version # type: ignore req = init_requirement("{0}{1}".format(name, version)) - if extras: - req.extras += tuple(extras) + if req and extras and req.extras and isinstance(req.extras, tuple): + if isinstance(extras, six.string_types): + req.extras = (extras) + tuple(["{0}".format(xtra) for xtra in req.extras]) + elif isinstance(extras, (tuple, list)): + req.extras += tuple(extras) creation_args["req"] = req return cls(**creation_args) # type: ignore diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 000b9557..6a68d6dc 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -6,14 +6,12 @@ import os import re import string import sys - from collections import defaultdict from itertools import chain, groupby from operator import attrgetter import six import tomlkit - from attr import validators from first import first from packaging.markers import InvalidMarker, Marker, Op, Value, Variable @@ -25,22 +23,37 @@ from vistir.compat import lru_cache from vistir.misc import dedup from vistir.path import is_valid_url - -from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri - from ..environment import MYPY_RUNNING +from ..utils import SCHEME_LIST, VCS_LIST, add_ssh_scheme_to_git_uri, is_star if MYPY_RUNNING: - from typing import Union, Optional, List, Set, Any, TypeVar, Tuple, Sequence, Dict, Text, AnyStr, Match, Iterable + from typing import ( + Union, + Optional, + List, + Set, + Any, + TypeVar, + Tuple, + Sequence, + Dict, + Text, + AnyStr, + Match, + Iterable, + ) from attr import _ValidatorType from packaging.requirements import Requirement as PackagingRequirement from pkg_resources import Requirement as PkgResourcesRequirement from pkg_resources.extern.packaging.markers import ( - Op as PkgResourcesOp, Variable as PkgResourcesVariable, - Value as PkgResourcesValue, Marker as PkgResourcesMarker + Op as PkgResourcesOp, + Variable as PkgResourcesVariable, + Value as PkgResourcesValue, + Marker as PkgResourcesMarker, ) from pip_shims.shims import Link from vistir.compat import Path + _T = TypeVar("_T") TMarker = Union[Marker, PkgResourcesMarker] TVariable = TypeVar("TVariable", PkgResourcesVariable, Variable) @@ -65,7 +78,13 @@ NAME_RE = re.compile(NAME_WITH_EXTRAS) SUBDIR_RE = r"(?:[&#]subdirectory=(?P.*))" URL_NAME = r"(?:#egg={0})".format(NAME_WITH_EXTRAS) REF_RE = r"(?:@(?P{0}+)?)".format(REF) -URL = r"(?P[^ ]+://)(?:(?P[^ ]+?\.?{0}+(?P:\d+)?))?(?P[:/])(?P[^ @]+){1}?".format(ALPHA_NUMERIC, REF_RE) +PATH_RE = r"(?P[:/])(?P[^ @]+){0}?".format(REF_RE) +PASS_RE = r"(?:(?<=:)(?P[^ ]+))" +AUTH_RE = r"(?:(?P[^ ]+)[:@]{0}?@)".format(PASS_RE) +HOST_RE = r"(?:{0}?(?P[^ ]+?\.?{1}+(?P:\d+)?))?".format( + AUTH_RE, ALPHA_NUMERIC +) +URL = r"(?P[^ ]+://){0}{1}".format(HOST_RE, PATH_RE) URL_RE = re.compile(r"{0}(?:{1}?{2}?)?".format(URL, URL_NAME, SUBDIR_RE)) DIRECT_URL_RE = re.compile(r"{0}\s?@\s?{1}".format(NAME_WITH_EXTRAS, URL)) @@ -88,6 +107,7 @@ def create_link(link): if not isinstance(link, six.string_types): raise TypeError("must provide a string to instantiate a new link") from pip_shims.shims import Link + return Link(link) @@ -111,6 +131,7 @@ def init_requirement(name): if not isinstance(name, six.string_types): raise TypeError("must supply a name to generate a requirement") from pkg_resources import Requirement + req = Requirement.parse(name) req.vcs = None req.local_file = None @@ -139,6 +160,7 @@ def parse_extras(extras_str): """ from pkg_resources import Requirement + extras = Requirement.parse("fakepkg{0}".format(extras_to_string(extras_str))).extras return sorted(dedup([extra.lower() for extra in extras])) @@ -166,7 +188,7 @@ def build_vcs_uri( name=None, # type: Optional[S] ref=None, # type: Optional[S] subdirectory=None, # type: Optional[S] - extras=None # type: Optional[Iterable[S]] + extras=None, # type: Optional[Iterable[S]] ): # type: (...) -> STRING_TYPE if extras is None: @@ -203,11 +225,17 @@ def convert_direct_url_to_url(direct_url): url_match = URL_RE.match(direct_url) if url_match or is_valid_url(direct_url): return direct_url - match_dict = {} # type: Dict[STRING_TYPE, Union[Tuple[STRING_TYPE, ...], STRING_TYPE]] + match_dict = ( + {} + ) # type: Dict[STRING_TYPE, Union[Tuple[STRING_TYPE, ...], STRING_TYPE]] if direct_match is not None: match_dict = direct_match.groupdict() # type: ignore if not match_dict: - raise ValueError("Failed converting value to normal URL, is it a direct URL? {0!r}".format(direct_url)) + raise ValueError( + "Failed converting value to normal URL, is it a direct URL? {0!r}".format( + direct_url + ) + ) url_segments = [match_dict.get(s) for s in ("scheme", "host", "path", "pathsep")] url = "" # type: STRING_TYPE url = "".join([s for s in url_segments if s is not None]) # type: ignore @@ -217,7 +245,7 @@ def convert_direct_url_to_url(direct_url): ref=match_dict.get("ref"), name=match_dict.get("name"), extras=match_dict.get("extras"), - subdirectory=match_dict.get("subdirectory") + subdirectory=match_dict.get("subdirectory"), ) return new_url @@ -334,6 +362,7 @@ def _strip_extras_markers(marker): def get_setuptools_version(): # type: () -> Optional[STRING_TYPE] import pkg_resources + setuptools_dist = pkg_resources.get_distribution( pkg_resources.Requirement("setuptools") ) @@ -364,6 +393,7 @@ def get_pyproject(path): if not path: return from vistir.compat import Path + if not isinstance(path, Path): path = Path(path) if not path.is_dir(): @@ -387,10 +417,7 @@ def get_pyproject(path): else: requires = ["setuptools>=40.8", "wheel"] backend = get_default_pyproject_backend() - build_system = { - "requires": requires, - "build-backend": backend - } + build_system = {"requires": requires, "build-backend": backend} pyproject_data["build_system"] = build_system else: requires = build_system.get("requires", ["setuptools>=40.8", "wheel"]) @@ -480,14 +507,14 @@ def key_from_ireq(ireq): def key_from_req(req): """Get an all-lowercase version of the requirement's name.""" - if hasattr(req, 'key'): + if hasattr(req, "key"): # from pkg_resources, such as installed dists for pip-sync key = req.key else: # from packaging, such as install requirements from requirements.txt key = req.name - key = key.replace('_', '-').lower() + key = key.replace("_", "-").lower() return key @@ -529,17 +556,17 @@ def format_requirement(ireq): """ if ireq.editable: - line = '-e {}'.format(ireq.link) + line = "-e {}".format(ireq.link) else: line = _requirement_to_str_lowercase_name(ireq.req) if str(ireq.req.marker) != str(ireq.markers): if not ireq.req.marker: - line = '{}; {}'.format(line, ireq.markers) + line = "{}; {}".format(line, ireq.markers) else: name, markers = line.split(";", 1) markers = markers.strip() - line = '{}; ({}) and ({})'.format(name, markers, ireq.markers) + line = "{}; ({}) and ({})".format(name, markers, ireq.markers) return line @@ -552,7 +579,7 @@ def format_specifier(ireq): # TODO: Ideally, this is carried over to the pip library itself specs = ireq.specifier._specs if ireq.req is not None else [] specs = sorted(specs, key=lambda x: x._spec[1]) - return ','.join(str(s) for s in specs) or '' + return ",".join(str(s) for s in specs) or "" def get_pinned_version(ireq): @@ -579,9 +606,7 @@ def get_pinned_version(ireq): try: specifier = ireq.specifier except AttributeError: - raise TypeError("Expected InstallRequirement, not {}".format( - type(ireq).__name__, - )) + raise TypeError("Expected InstallRequirement, not {}".format(type(ireq).__name__)) if ireq.editable: raise ValueError("InstallRequirement is editable") @@ -591,10 +616,8 @@ def get_pinned_version(ireq): raise ValueError("InstallRequirement has multiple specifications") op, version = next(iter(specifier._specs))._spec - if op not in ('==', '===') or version.endswith('.*'): - raise ValueError("InstallRequirement not pinned (is {0!r})".format( - op + version, - )) + if op not in ("==", "===") or version.endswith(".*"): + raise ValueError("InstallRequirement not pinned (is {0!r})".format(op + version)) return version @@ -630,7 +653,7 @@ def as_tuple(ireq): """ if not is_pinned_requirement(ireq): - raise TypeError('Expected a pinned InstallRequirement, got {}'.format(ireq)) + raise TypeError("Expected a pinned InstallRequirement, got {}".format(ireq)) name = key_from_req(ireq.req) version = first(ireq.specifier._specs)._spec[1] @@ -692,9 +715,9 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): if keyval is None: if key is None: - keyval = (lambda v: v) + keyval = lambda v: v else: - keyval = (lambda v: (key(v), v)) + keyval = lambda v: (key(v), v) if unique: return dict(keyval(v) for v in values) @@ -718,7 +741,7 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): def name_from_req(req): """Get the name of the requirement""" - if hasattr(req, 'project_name'): + if hasattr(req, "project_name"): # from pkg_resources, such as installed dists for pip-sync return req.project_name else: @@ -748,6 +771,7 @@ def make_install_requirement(name, version, extras, markers, constraint=False): # If no extras are specified, the extras string is blank from pip_shims.shims import install_req_from_line + extras_string = "" if extras: # Sort extras for stability @@ -755,12 +779,13 @@ def make_install_requirement(name, version, extras, markers, constraint=False): if not markers: return install_req_from_line( - str('{}{}=={}'.format(name, extras_string, version)), - constraint=constraint) + str("{}{}=={}".format(name, extras_string, version)), constraint=constraint + ) else: return install_req_from_line( - str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), - constraint=constraint) + str("{}{}=={}; {}".format(name, extras_string, version, str(markers))), + constraint=constraint, + ) def version_from_ireq(ireq): @@ -778,9 +803,10 @@ def version_from_ireq(ireq): def clean_requires_python(candidates): """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" all_candidates = [] - sys_version = '.'.join(map(str, sys.version_info[:3])) + sys_version = ".".join(map(str, sys.version_info[:3])) from packaging.version import parse as parse_version - py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', sys_version)) + + py_version = parse_version(os.environ.get("PIP_PYTHON_VERSION", sys_version)) for c in candidates: from_location = attrgetter("location.requires_python") requires_python = getattr(c, "requires_python", from_location(c)) @@ -788,7 +814,9 @@ def clean_requires_python(candidates): # Old specifications had people setting this to single digits # which is effectively the same as '>=digit,=2.6"')` - marker_key = Variable('python_version') + marker_key = Variable("python_version") for spec in specifierset: operator, val = spec._spec cleaned_val = Value(val).serialize().replace('"', "") spec_dict[Op(operator).serialize()].add(cleaned_val) - marker_str = ' and '.join([ - "{0}{1}'{2}'".format(marker_key.serialize(), op, ','.join(vals)) - for op, vals in spec_dict.items() - ]) - marker_to_add = PackagingRequirement('fakepkg; {0}'.format(marker_str)).marker + marker_str = " and ".join( + [ + "{0}{1}'{2}'".format(marker_key.serialize(), op, ",".join(vals)) + for op, vals in spec_dict.items() + ] + ) + marker_to_add = PackagingRequirement("fakepkg; {0}".format(marker_str)).marker return marker_to_add @@ -851,6 +882,7 @@ def get_name_variants(pkg): raise TypeError("must provide a string to derive package names") from pkg_resources import safe_name from packaging.utils import canonicalize_name + pkg = pkg.lower() names = {safe_name(pkg), canonicalize_name(pkg), pkg.replace("-", "_")} return names diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index a3ddfdde..0aba3a43 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -4,27 +4,53 @@ from __future__ import absolute_import, print_function import contextlib import logging import os - -import six import sys -import tomlkit -import vistir - -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) # type: ignore # noqa -six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) # type: ignore # noqa -six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # type: ignore # noqa -six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) # type: ignore # noqa -from six.moves import Mapping, Sequence, Set, ItemsView # type: ignore # noqa -from six.moves.urllib.parse import urlparse, urlsplit, urlunparse import pip_shims.shims +import six +import tomlkit +import vistir +from six.moves.urllib.parse import urlparse, urlsplit, urlunparse from vistir.compat import Path -from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir +from vistir.path import create_tracked_tempdir, ensure_mkdir_p, is_valid_url from .environment import MYPY_RUNNING +# fmt: off +six.add_move( + six.MovedAttribute("Mapping", "collections", "collections.abc") +) # type: ignore # noqa # isort:skip +six.add_move( + six.MovedAttribute("Sequence", "collections", "collections.abc") +) # type: ignore # noqa # isort:skip +six.add_move( + six.MovedAttribute("Set", "collections", "collections.abc") +) # type: ignore # noqa # isort:skip +six.add_move( + six.MovedAttribute("ItemsView", "collections", "collections.abc") +) # type: ignore # noqa +from six.moves import ItemsView, Mapping, Sequence, Set # type: ignore # noqa # isort:skip +# fmt: on + + if MYPY_RUNNING: - from typing import Dict, Any, Optional, Union, Tuple, List, Iterable, Generator, Text + from typing import ( + Dict, + Any, + Optional, + Union, + Tuple, + List, + Iterable, + Generator, + Text, + TypeVar, + ) + + STRING_TYPE = Union[bytes, str, Text] + S = TypeVar("S", bytes, str, Text) + PipfileEntryType = Union[STRING_TYPE, bool, Tuple[STRING_TYPE], List[STRING_TYPE]] + PipfileType = Union[STRING_TYPE, Dict[STRING_TYPE, PipfileEntryType]] VCS_LIST = ("git", "svn", "hg", "bzr") @@ -74,7 +100,7 @@ VCS_SCHEMES = [ def is_installable_dir(path): - # type: (Text) -> bool + # type: (STRING_TYPE) -> bool if pip_shims.shims.is_installable_dir(path): return True pyproject_path = os.path.join(path, "pyproject.toml") @@ -88,7 +114,7 @@ def is_installable_dir(path): def strip_ssh_from_git_uri(uri): - # type: (Text) -> Text + # type: (S) -> S """Return git+ssh:// formatted URI to git+git@ format""" if isinstance(uri, six.string_types): if "git+ssh://" in uri: @@ -105,7 +131,7 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): - # type: (Text) -> Text + # type: (S) -> S """Cleans VCS uris from pip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does @@ -120,7 +146,7 @@ def add_ssh_scheme_to_git_uri(uri): def is_vcs(pipfile_entry): - # type: (Union[Text, Dict[Text, Union[Text, bool, Tuple[Text], List[Text]]]]) -> bool + # type: (PipfileType) -> bool """Determine if dictionary entry from Pipfile is for a vcs dependency.""" if isinstance(pipfile_entry, Mapping): return any(key for key in pipfile_entry.keys() if key in VCS_LIST) @@ -135,7 +161,7 @@ def is_vcs(pipfile_entry): def is_editable(pipfile_entry): - # type: (Union[Text, Dict[Text, Union[Text, bool, Tuple[Text], List[Text]]]]) -> bool + # type: (PipfileType) -> bool if isinstance(pipfile_entry, Mapping): return pipfile_entry.get("editable", False) is True if isinstance(pipfile_entry, six.string_types): @@ -144,7 +170,7 @@ def is_editable(pipfile_entry): def multi_split(s, split): - # type: (Text, Iterable[Text]) -> List[Text] + # type: (S, Iterable[S]) -> List[S] """Splits on multiple given separators.""" for r in split: s = s.replace(r, "|") @@ -152,14 +178,14 @@ def multi_split(s, split): def is_star(val): - # type: (Union[Text, Dict[Text, Union[Text, bool, Tuple[Text], List[Text]]]]) -> bool + # type: (PipfileType) -> bool return (isinstance(val, six.string_types) and val == "*") or ( isinstance(val, Mapping) and val.get("version", "") == "*" ) def convert_entry_to_path(path): - # type: (Dict[Text, Union[Text, bool, Tuple[Text], List[Text]]]) -> Text + # type: (Dict[S, Union[S, bool, Tuple[S], List[S]]]) -> S """Convert a pipfile entry to a string""" if not isinstance(path, Mapping): @@ -177,7 +203,7 @@ def convert_entry_to_path(path): def is_installable_file(path): - # type: (Union[Text, Dict[Text, Union[Text, bool, Tuple[Text], List[Text]]]]) -> bool + # type: (PipfileType) -> bool """Determine if a path can potentially be installed""" from packaging import specifiers @@ -196,7 +222,11 @@ def is_installable_file(path): return False parsed = urlparse(path) - is_local = (not parsed.scheme or parsed.scheme == "file" or (len(parsed.scheme) == 1 and os.name == "nt")) + is_local = ( + not parsed.scheme + or parsed.scheme == "file" + or (len(parsed.scheme) == 1 and os.name == "nt") + ) if parsed.scheme and parsed.scheme == "file": path = vistir.compat.fs_decode(vistir.path.url_to_path(path)) normalized_path = vistir.path.normalize_path(path) @@ -204,7 +234,9 @@ def is_installable_file(path): return False is_archive = pip_shims.shims.is_archive_file(normalized_path) - is_local_project = os.path.isdir(normalized_path) and is_installable_dir(normalized_path) + is_local_project = os.path.isdir(normalized_path) and is_installable_dir( + normalized_path + ) if is_local and is_local_project or is_archive: return True @@ -217,11 +249,13 @@ def is_installable_file(path): def get_dist_metadata(dist): import pkg_resources from email.parser import FeedParser - if (isinstance(dist, pkg_resources.DistInfoDistribution) and - dist.has_metadata('METADATA')): - metadata = dist.get_metadata('METADATA') - elif dist.has_metadata('PKG-INFO'): - metadata = dist.get_metadata('PKG-INFO') + + if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata( + "METADATA" + ): + metadata = dist.get_metadata("METADATA") + elif dist.has_metadata("PKG-INFO"): + metadata = dist.get_metadata("PKG-INFO") else: metadata = "" @@ -231,7 +265,7 @@ def get_dist_metadata(dist): def get_setup_paths(base_path, subdirectory=None): - # type: (Text, Optional[Text]) -> Dict[Text, Optional[Text]] + # type: (S, Optional[S]) -> Dict[S, Optional[S]] if base_path is None: raise TypeError("must provide a path to derive setup paths from") setup_py = os.path.join(base_path, "setup.py") @@ -251,12 +285,12 @@ def get_setup_paths(base_path, subdirectory=None): return { "setup_py": setup_py if os.path.exists(setup_py) else None, "setup_cfg": setup_cfg if os.path.exists(setup_cfg) else None, - "pyproject_toml": pyproject_toml if os.path.exists(pyproject_toml) else None + "pyproject_toml": pyproject_toml if os.path.exists(pyproject_toml) else None, } def prepare_pip_source_args(sources, pip_args=None): - # type: (List[Dict[Text, Union[Text, bool]]], Optional[List[Text]]) -> List[Text] + # type: (List[Dict[S, Union[S, bool]]], Optional[List[S]]) -> List[S] if pip_args is None: pip_args = [] if sources: @@ -264,7 +298,9 @@ def prepare_pip_source_args(sources, pip_args=None): pip_args.extend(["-i", sources[0]["url"]]) # type: ignore # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): - pip_args.extend(["--trusted-host", urlparse(sources[0]["url"]).hostname]) # type: ignore + pip_args.extend( + ["--trusted-host", urlparse(sources[0]["url"]).hostname] + ) # type: ignore # Add additional sources as extra indexes. if len(sources) > 1: for source in sources[1:]: @@ -284,7 +320,7 @@ def _ensure_dir(path): @contextlib.contextmanager def ensure_setup_py(base): - # type: (Text) -> Generator[None, None, None] + # type: (STRING_TYPE) -> Generator[None, None, None] if not base: base = create_tracked_tempdir(prefix="requirementslib-setup") base_dir = Path(base) @@ -413,9 +449,7 @@ def get_path(root, path, default=_UNSET): cur = cur[seg] except (ValueError, KeyError, IndexError, TypeError): if not getattr(cur, "__iter__", None): - exc = TypeError( - "%r object is not indexable" % type(cur).__name__ - ) + exc = TypeError("%r object is not indexable" % type(cur).__name__) raise PathAccessError(exc, seg, path) except PathAccessError: if default is _UNSET: From 7fe11e73e54fab907521c3429297c592a98df119 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 4 Mar 2019 01:10:51 -0500 Subject: [PATCH 11/21] Add new file for requirementslib Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/url.py | 398 ++++++++++++++++++++ 1 file changed, 398 insertions(+) create mode 100644 pipenv/vendor/requirementslib/models/url.py diff --git a/pipenv/vendor/requirementslib/models/url.py b/pipenv/vendor/requirementslib/models/url.py new file mode 100644 index 00000000..8a5337ff --- /dev/null +++ b/pipenv/vendor/requirementslib/models/url.py @@ -0,0 +1,398 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import attr +import pip_shims.shims +from orderedmultidict import omdict +from six.moves.urllib.parse import quote_plus, unquote_plus +from urllib3 import util as urllib3_util +from urllib3.util import parse_url as urllib3_parse +from urllib3.util.url import Url + +from ..environment import MYPY_RUNNING + +if MYPY_RUNNING: + from typing import List, Tuple, Text, Union, TypeVar, Optional + from pip_shims.shims import Link + from vistir.compat import Path + + _T = TypeVar("_T") + STRING_TYPE = Union[bytes, str, Text] + S = TypeVar("S", bytes, str, Text) + + +def _get_parsed_url(url): + # type: (S) -> Url + """ + This is a stand-in function for `urllib3.util.parse_url` + + The orignal function doesn't handle special characters very well, this simply splits + out the authentication section, creates the parsed url, then puts the authentication + section back in, bypassing validation. + + :return: The new, parsed URL object + :rtype: :class:`~urllib3.util.url.Url` + """ + + try: + parsed = urllib3_parse(url) + except ValueError: + scheme, _, url = url.partition("://") + auth, _, url = url.rpartition("@") + url = "{scheme}://{url}".format(scheme=scheme, url=url) + parsed = urllib3_parse(url)._replace(auth=auth) + return parsed + + +def remove_password_from_url(url): + # type: (S) -> S + """ + Given a url, remove the password and insert 4 dashes + + :param url: The url to replace the authentication in + :type url: S + :return: The new URL without authentication + :rtype: S + """ + + parsed = _get_parsed_url(url) + if parsed.auth: + auth, _, _ = parsed.auth.partition(":") + return parsed._replace(auth="{auth}:----".format(auth=auth)).url + return parsed.url + + +@attr.s +class URI(object): + #: The target hostname, e.g. `amazon.com` + host = attr.ib(type=str) + #: The URI Scheme, e.g. `salesforce` + scheme = attr.ib(default="https", type=str) + #: The numeric port of the url if specified + port = attr.ib(default=None, type=int) + #: The url path, e.g. `/path/to/endpoint` + path = attr.ib(default="", type=str) + #: Query parameters, e.g. `?variable=value...` + query = attr.ib(default="", type=str) + #: URL Fragments, e.g. `#fragment=value` + fragment = attr.ib(default="", type=str) + #: Subdirectory fragment, e.g. `&subdirectory=blah...` + subdirectory = attr.ib(default="", type=str) + #: VCS ref this URI points at, if available + ref = attr.ib(default="", type=str) + #: The username if provided, parsed from `user:password@hostname` + username = attr.ib(default="", type=str) + #: Password parsed from `user:password@hostname` + password = attr.ib(default="", type=str, repr=False) + #: An orderedmultidict representing query fragments + query_dict = attr.ib(factory=omdict, type=omdict) + #: The name of the specified package in case it is a VCS URI with an egg fragment + name = attr.ib(default="", type=str) + #: Any extras requested from the requirement + extras = attr.ib(factory=tuple, type=tuple) + #: Whether the url was parsed as a direct pep508-style URL + is_direct_url = attr.ib(default=False, type=bool) + #: Whether the url was an implicit `git+ssh` url (passed as `git+git@`) + is_implicit_ssh = attr.ib(default=False, type=bool) + _auth = attr.ib(default=None, type=str, repr=False) + _fragment_dict = attr.ib(factory=dict, type=dict) + + def _parse_query(self): + # type: () -> URI + query = self.query if self.query is not None else "" + query_dict = omdict() + queries = query.split("&") + query_items = [] + for q in queries: + key, _, val = q.partition("=") + val = unquote_plus(val.replace("+", " ")) + query_items.append((key, val)) + query_dict.load(query_items) + return attr.evolve(self, query_dict=query_dict, query=query) + + def _parse_fragment(self): + # type: () -> URI + subdirectory = self.subdirectory if self.subdirectory else "" + fragment = self.fragment if self.fragment else "" + if self.fragment is None: + return self + fragments = self.fragment.split("&") + fragment_items = {} + name = self.name if self.name else "" + extras = self.extras + for q in fragments: + key, _, val = q.partition("=") + val = unquote_plus(val.replace("+", " ")) + fragment_items[key] = val + if key == "egg": + from .utils import parse_extras + + name, stripped_extras = pip_shims.shims._strip_extras(val) + if stripped_extras: + extras = tuple(parse_extras(stripped_extras)) + elif key == "subdirectory": + subdirectory = val + return attr.evolve( + self, + fragment_dict=fragment_items, + subdirectory=subdirectory, + fragment=fragment, + extras=extras, + name=name, + ) + + def _parse_auth(self): + # type: () -> URI + if self._auth: + username, _, password = self._auth.partition(":") + password = quote_plus(password) + return attr.evolve(self, username=username, password=password) + return self + + def get_password(self, unquote=False, include_token=True): + # type: (bool, bool) -> str + password = self.password + if password and unquote: + password = unquote_plus(password) + else: + password = "" + return password + + @staticmethod + def parse_subdirectory(url_part): + # type: (str) -> Tuple[str, Optional[str]] + subdir = None + if "&subdirectory" in url_part: + url_part, _, subdir = url_part.rpartition("&") + subdir = "&{0}".format(subdir.strip()) + return url_part.strip(), subdir + + @classmethod + def parse(cls, url): + # type: (S) -> URI + from .utils import DIRECT_URL_RE, split_ref_from_uri + + is_direct_url = False + name_with_extras = None + is_implicit_ssh = url.strip().startswith("git+git@") + if is_implicit_ssh: + from ..utils import add_ssh_scheme_to_git_uri + + url = add_ssh_scheme_to_git_uri(url) + direct_match = DIRECT_URL_RE.match(url) + if direct_match is not None: + is_direct_url = True + name_with_extras, _, url = url.partition("@") + name_with_extras = name_with_extras.strip() + url, ref = split_ref_from_uri(url.strip()) + if "file:/" in url and "file:///" not in url: + url = url.replace("file:/", "file:///") + parsed = _get_parsed_url(url) + if not (parsed.scheme and parsed.host and parsed.path): + # check if this is a file uri + if not ( + parsed.scheme + and parsed.path + and (parsed.scheme == "file" or parsed.scheme.endswith("+file")) + ): + raise ValueError("Failed parsing URL {0!r} - Not a valid url".format(url)) + parsed_dict = dict(parsed._asdict()).copy() + parsed_dict["is_direct_url"] = is_direct_url + parsed_dict["is_implicit_ssh"] = is_implicit_ssh + if name_with_extras: + fragment = "" + if parsed_dict["fragment"] is not None: + fragment = "{0}".format(parsed_dict["fragment"]) + elif "&subdirectory" in parsed_dict["path"]: + path, fragment = cls.parse_subdirectory(parsed_dict["path"]) + parsed_dict["path"] = path + elif ref is not None and "&subdirectory" in ref: + ref, fragment = cls.parse_subdirectory(ref) + parsed_dict["fragment"] = "egg={0}{1}".format(name_with_extras, fragment) + if ref is not None: + parsed_dict["ref"] = ref.strip() + return cls(**parsed_dict)._parse_auth()._parse_query()._parse_fragment() + + def to_string( + self, + escape_password=True, # type: bool + unquote=True, # type: bool + direct=None, # type: Optional[bool] + strip_ssh=False, # type: bool + strip_ref=False, # type: bool + strip_name=False, # type: bool + strip_subdir=False, # type: bool + ): + # type: (...) -> str + """ + Converts the current URI to a string, unquoting or escaping the password as needed + + :param escape_password: Whether to replace password with ``----``, default True + :param escape_password: bool, optional + :param unquote: Whether to unquote url-escapes in the password, default False + :param unquote: bool, optional + :param bool direct: Whether to format as a direct URL + :param bool strip_ssh: Whether to strip the SSH scheme from the url (git only) + :param bool strip_ref: Whether to drop the VCS ref (if present) + :param bool strip_name: Whether to drop the name and extras (if present) + :param bool strip_subdir: Whether to drop the subdirectory (if present) + :return: The reconstructed string representing the URI + :rtype: str + """ + + if direct is None: + direct = self.is_direct_url + if escape_password: + password = "----" if (self.password or self.username) else "" + else: + password = self.get_password(unquote=unquote) + auth = "" + if self.username: + if password: + auth = "{self.username}:{password}@".format(password=password, self=self) + else: + auth = "{self.username}@".format(self=self) + query = "" + if self.query: + query = "{query}?{self.query}".format(query=query, self=self) + if not direct: + if self.name and not strip_name: + fragment = "#egg={self.name_with_extras}".format(self=self) + elif not strip_name and ( + self.extras and self.scheme and self.scheme.startswith("file") + ): + from .utils import extras_to_string + + fragment = extras_to_string(self.extras) + else: + fragment = "" + query = "{query}{fragment}".format(query=query, fragment=fragment) + if self.subdirectory and not strip_subdir: + query = "{query}&subdirectory={self.subdirectory}".format( + query=query, self=self + ) + host_port_path = self.get_host_port_path(strip_ref=strip_ref) + url = "{self.scheme}://{auth}{host_port_path}{query}".format( + self=self, auth=auth, host_port_path=host_port_path, query=query + ) + if strip_ssh: + from ..utils import strip_ssh_from_git_uri + + url = strip_ssh_from_git_uri(url) + if self.name and direct and not strip_name: + return "{self.name_with_extras}@ {url}".format(self=self, url=url) + return url + + def get_host_port_path(self, strip_ref=False): + # type: (bool) -> str + host = self.host if self.host else "" + if self.port: + host = "{host}:{self.port!s}".format(host=host, self=self) + path = "{self.path}".format(self=self) + if self.ref and not strip_ref: + path = "{path}@{self.ref}".format(path=path, self=self) + return "{host}{path}".format(host=host, path=path) + + @property + def name_with_extras(self): + # type: () -> str + from .utils import extras_to_string + + if not self.name: + return "" + extras = extras_to_string(self.extras) + return "{self.name}{extras}".format(self=self, extras=extras) + + @property + def as_link(self): + # type: () -> Link + link = pip_shims.shims.Link( + self.to_string(escape_password=False, strip_ssh=False, direct=False) + ) + return link + + @property + def bare_url(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + strip_ref=True, + strip_subdir=True, + ) + + @property + def url_without_fragment_or_ref(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + strip_ref=True, + ) + + @property + def url_without_fragment(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + ) + + @property + def url_without_ref(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_ref=True, + ) + + @property + def base_url(self): + # type: () -> str + return self.to_string( + escape_password=False, strip_ssh=self.is_implicit_ssh, direct=False + ) + + @property + def full_url(self): + # type: () -> str + return self.to_string(escape_password=False, strip_ssh=False, direct=False) + + @property + def safe_string(self): + # type: () -> str + return self.to_string(escape_password=True, unquote=True) + + @property + def unsafe_string(self): + # type: () -> str + return self.to_string(escape_password=False, unquote=True) + + @property + def uri_escape(self): + # type: () -> str + return self.to_string(escape_password=False, unquote=False) + + @property + def is_vcs(self): + # type: () -> bool + from ..utils import VCS_SCHEMES + + return self.scheme in VCS_SCHEMES + + @property + def is_file_url(self): + # type: () -> bool + return all([self.scheme, self.scheme == "file"]) + + def __str__(self): + # type: () -> str + return self.to_string(escape_password=True, unquote=True) From a1cb576189cbfdc66f67753480d8f50d4cec188e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 4 Mar 2019 09:35:02 -0500 Subject: [PATCH 12/21] vendor orderedmultidict Signed-off-by: Dan Ryan --- pipenv/vendor/orderedmultidict/LICENSE | 31 + pipenv/vendor/orderedmultidict/__init__.py | 21 + pipenv/vendor/orderedmultidict/itemlist.py | 157 ++++ .../orderedmultidict/orderedmultidict.py | 811 ++++++++++++++++++ pipenv/vendor/vendor.txt | 1 + 5 files changed, 1021 insertions(+) create mode 100755 pipenv/vendor/orderedmultidict/LICENSE create mode 100755 pipenv/vendor/orderedmultidict/__init__.py create mode 100755 pipenv/vendor/orderedmultidict/itemlist.py create mode 100755 pipenv/vendor/orderedmultidict/orderedmultidict.py diff --git a/pipenv/vendor/orderedmultidict/LICENSE b/pipenv/vendor/orderedmultidict/LICENSE new file mode 100755 index 00000000..fd832f40 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/LICENSE @@ -0,0 +1,31 @@ +Build Amazing Things. + +*** + +### Unlicense + +This is free and unencumbered software released into the public\ +domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or distribute\ +this software, either in source code form or as a compiled binary, for any\ +purpose, commercial or non-commercial, and by any means. + +In jurisdictions that recognize copyright laws, the author or authors of\ +this software dedicate any and all copyright interest in the software to the\ +public domain. We make this dedication for the benefit of the public at\ +large and to the detriment of our heirs and successors. We intend this\ +dedication to be an overt act of relinquishment in perpetuity of all\ +present and future rights to this software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF\ +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\ +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\ +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\ +SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR\ +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT\ +OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\ +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\ +SOFTWARE. + +For more information, please refer to diff --git a/pipenv/vendor/orderedmultidict/__init__.py b/pipenv/vendor/orderedmultidict/__init__.py new file mode 100755 index 00000000..4f0ce2f7 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) + +from __future__ import absolute_import + +from .orderedmultidict import * # noqa + +__title__ = 'orderedmultidict' +__version__ = '1.0' +__author__ = 'Ansgar Grunseid' +__contact__ = 'grunseid@gmail.com' +__license__ = 'Unlicense' +__url__ = 'https://github.com/gruns/orderedmultidict' diff --git a/pipenv/vendor/orderedmultidict/itemlist.py b/pipenv/vendor/orderedmultidict/itemlist.py new file mode 100755 index 00000000..e9e96c72 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/itemlist.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) + +from __future__ import absolute_import + +from six.moves import zip_longest + +_absent = object() # Marker that means no parameter was provided. + + +class itemnode(object): + + """ + Dictionary key:value items wrapped in a node to be members of itemlist, the + doubly linked list defined below. + """ + + def __init__(self, prev=None, next=None, key=_absent, value=_absent): + self.prev = prev + self.next = next + self.key = key + self.value = value + + +class itemlist(object): + + """ + Doubly linked list of itemnodes. + + This class is used as the key:value item storage of orderedmultidict. + Methods below were only added as needed for use with orderedmultidict, so + some otherwise common list methods may be missing. + """ + + def __init__(self, items=[]): + self.root = itemnode() + self.root.next = self.root.prev = self.root + self.size = 0 + + for key, value in items: + self.append(key, value) + + def append(self, key, value): + tail = self.root.prev if self.root.prev is not self.root else self.root + node = itemnode(tail, self.root, key=key, value=value) + tail.next = node + self.root.prev = node + self.size += 1 + return node + + def removenode(self, node): + node.prev.next = node.next + node.next.prev = node.prev + self.size -= 1 + return self + + def clear(self): + for node, key, value in self: + self.removenode(node) + return self + + def items(self): + return list(self.iteritems()) + + def keys(self): + return list(self.iterkeys()) + + def values(self): + return list(self.itervalues()) + + def iteritems(self): + for node, key, value in self: + yield key, value + + def iterkeys(self): + for node, key, value in self: + yield key + + def itervalues(self): + for node, key, value in self: + yield value + + def reverse(self): + for node, key, value in self: + node.prev, node.next = node.next, node.prev + self.root.prev, self.root.next = self.root.next, self.root.prev + return self + + def __len__(self): + return self.size + + def __iter__(self): + current = self.root.next + while current and current is not self.root: + # Record current.next here in case current.next changes after the + # yield and before we return for the next iteration. For example, + # methods like reverse() will change current.next() before yield + # gets executed again. + nextnode = current.next + yield current, current.key, current.value + current = nextnode + + def __contains__(self, item): + """ + Params: + item: Can either be a (key,value) tuple or an itemnode reference. + """ + node = key = value = _absent + if hasattr(item, '__len__') and callable(item.__len__): + if len(item) == 2: + key, value = item + elif len(item) == 3: + node, key, value = item + else: + node = item + + if node is not _absent or _absent not in [key, value]: + for selfnode, selfkey, selfvalue in self: + if ((node is _absent and key == selfkey and value == selfvalue) + or (node is not _absent and node == selfnode)): + return True + return False + + def __getitem__(self, index): + # Only support direct access to the first or last element, as this is + # all orderedmultidict needs for now. + if index == 0 and self.root.next is not self.root: + return self.root.next + elif index == -1 and self.root.prev is not self.root: + return self.root.prev + raise IndexError(index) + + def __delitem__(self, index): + self.removenode(self[index]) + + def __eq__(self, other): + for (n1, key1, value1), (n2, key2, value2) in zip_longest(self, other): + if key1 != key2 or value1 != value2: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __nonzero__(self): + return self.size > 0 + + def __str__(self): + return '[%s]' % self.items() diff --git a/pipenv/vendor/orderedmultidict/orderedmultidict.py b/pipenv/vendor/orderedmultidict/orderedmultidict.py new file mode 100755 index 00000000..924dd8d2 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/orderedmultidict.py @@ -0,0 +1,811 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) + +from __future__ import absolute_import + +from itertools import chain +from collections import MutableMapping + +import six +from six.moves import map, zip_longest + +from .itemlist import itemlist + +try: + from collections import OrderedDict as odict # Python 2.7 and later. +except ImportError: + from ordereddict import OrderedDict as odict # Python 2.6 and earlier. + +import sys +items_attr = 'items' if sys.version_info[0] >= 3 else 'iteritems' + +_absent = object() # Marker that means no parameter was provided. + + +def callable_attr(obj, attr): + return hasattr(obj, attr) and callable(getattr(obj, attr)) + + +# +# TODO(grun): Create a subclass of list that values(), getlist(), allitems(), +# etc return that the user can manipulate directly to control the omdict() +# object. +# +# For example, users should be able to do things like +# +# omd = omdict([(1,1), (1,11)]) +# omd.values(1).append('sup') +# omd.allitems() == [(1,1), (1,11), (1,'sup')] +# omd.values(1).remove(11) +# omd.allitems() == [(1,1), (1,'sup')] +# omd.values(1).extend(['two', 'more']) +# omd.allitems() == [(1,1), (1,'sup'), (1,'two'), (1,'more')] +# +# or +# +# omd = omdict([(1,1), (1,11)]) +# omd.allitems().extend([(2,2), (2,22)]) +# omd.allitems() == [(1,1), (1,11), (2,2), (2,22)]) +# +# or +# +# omd = omdict() +# omd.values(1) = [1, 11] +# omd.allitems() == [(1,1), (1,11)] +# omd.values(1) = list(map(lambda i: i * -10, omd.values(1))) +# omd.allitems() == [(1,-10), (1,-110)] +# omd.allitems() = filter(lambda (k,v): v > -100, omd.allitems()) +# omd.allitems() == [(1,-10)] +# +# etc. +# +# To accomplish this, subclass list in such a manner that each list element is +# really a two tuple, where the first tuple value is the actual value and the +# second tuple value is a reference to the itemlist node for that value. Users +# only interact with the first tuple values, the actual values, but behind the +# scenes when an element is modified, deleted, inserted, etc, the according +# itemlist nodes are modified, deleted, inserted, etc accordingly. In this +# manner, users can manipulate omdict objects directly through direct list +# manipulation. +# +# Once accomplished, some methods become redundant and should be removed in +# favor of the more intuitive direct value list manipulation. Such redundant +# methods include getlist() (removed in favor of values()?), addlist(), and +# setlist(). +# +# With the removal of many of the 'list' methods, think about renaming all +# remaining 'list' methods to 'values' methods, like poplist() -> popvalues(), +# poplistitem() -> popvaluesitem(), etc. This would be an easy switch for most +# methods, but wouldn't fit others so well. For example, iterlists() would +# become itervalues(), a name extremely similar to iterallvalues() but quite +# different in function. +# + + +class omdict(MutableMapping): + + """ + Ordered Multivalue Dictionary. + + A multivalue dictionary is a dictionary that can store multiple values per + key. An ordered multivalue dictionary is a multivalue dictionary that + retains the order of insertions and deletions. + + Internally, items are stored in a doubly linked list, self._items. A + dictionary, self._map, is also maintained and stores an ordered list of + linked list node references, one for each value associated with that key. + + Standard dict methods interact with the first value associated with a given + key. This means that omdict retains method parity with dict, and a dict + object can be replaced with an omdict object and all interaction will + behave identically. All dict methods that retain parity with omdict are: + + get(), setdefault(), pop(), popitem(), + clear(), copy(), update(), fromkeys(), len() + __getitem__(), __setitem__(), __delitem__(), __contains__(), + items(), keys(), values(), iteritems(), iterkeys(), itervalues(), + + Optional parameters have been added to some dict methods, but because the + added parameters are optional, existing use remains unaffected. An optional + parameter has been added to these methods: + + items(), values(), iteritems(), itervalues() + + New methods have also been added to omdict. Methods with 'list' in their + name interact with lists of values, and methods with 'all' in their name + interact with all items in the dictionary, including multiple items with + the same key. + + The new omdict methods are: + + load(), size(), reverse(), + getlist(), add(), addlist(), set(), setlist(), setdefaultlist(), + poplist(), popvalue(), popvalues(), popitem(), poplistitem(), + allitems(), allkeys(), allvalues(), lists(), listitems(), + iterallitems(), iterallkeys(), iterallvalues(), iterlists(), + iterlistitems() + + Explanations and examples of the new methods above can be found in the + function comments below and online at + + https://github.com/gruns/orderedmultidict + + Additional omdict information and documentation can also be found at the + above url. + """ + + def __init__(self, *args, **kwargs): + # Doubly linked list of itemnodes. Each itemnode stores a key:value + # item. + self._items = itemlist() + + # Ordered dictionary of keys and itemnode references. Each itemnode + # reference points to one of that keys values. + self._map = odict() + + self.load(*args, **kwargs) + + def load(self, *args, **kwargs): + """ + Clear all existing key:value items and import all key:value items from + . If multiple values exist for the same key in , they + are all be imported. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.load([(4,4), (4,44), (5,5)]) + omd.allitems() == [(4,4), (4,44), (5,5)] + + Returns: . + """ + self.clear() + self.updateall(*args, **kwargs) + return self + + def copy(self): + return self.__class__(self.allitems()) + + def clear(self): + self._map.clear() + self._items.clear() + + def size(self): + """ + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.size() == 5 + + Returns: Total number of items, including multiple items with the same + key. + """ + return len(self._items) + + @classmethod + def fromkeys(cls, iterable, value=None): + return cls([(key, value) for key in iterable]) + + def has_key(self, key): + return key in self + + def update(self, *args, **kwargs): + self._update_updateall(True, *args, **kwargs) + + def updateall(self, *args, **kwargs): + """ + Update this dictionary with the items from , replacing + existing key:value items with shared keys before adding new key:value + items. + + Example: + omd = omdict([(1,1), (2,2)]) + omd.updateall([(2,'two'), (1,'one'), (2,222), (1,111)]) + omd.allitems() == [(1, 'one'), (2, 'two'), (2, 222), (1, 111)] + + Returns: . + """ + self._update_updateall(False, *args, **kwargs) + return self + + def _update_updateall(self, replace_at_most_one, *args, **kwargs): + # Bin the items in and into or + # . Items in are new values to replace old + # values for a given key, and items in are new items to be + # added. + replacements, leftovers = dict(), [] + for mapping in chain(args, [kwargs]): + self._bin_update_items( + self._items_iterator(mapping), replace_at_most_one, + replacements, leftovers) + + # First, replace existing values for each key. + for key, values in six.iteritems(replacements): + self.setlist(key, values) + # Then, add the leftover items to the end of the list of all items. + for key, value in leftovers: + self.add(key, value) + + def _bin_update_items(self, items, replace_at_most_one, + replacements, leftovers): + """ + are modified directly, ala pass by + reference. + """ + for key, value in items: + # If there are existing items with key that have yet to be + # marked for replacement, mark that item's value to be replaced by + # by appending it to . + if key in self and key not in replacements: + replacements[key] = [value] + elif (key in self and not replace_at_most_one and + len(replacements[key]) < len(self.values(key))): + replacements[key].append(value) + else: + if replace_at_most_one: + replacements[key] = [value] + else: + leftovers.append((key, value)) + + def _items_iterator(self, container): + cont = container + iterator = iter(cont) + if callable_attr(cont, 'iterallitems'): + iterator = cont.iterallitems() + elif callable_attr(cont, 'allitems'): + iterator = iter(cont.allitems()) + elif callable_attr(cont, 'iteritems'): + iterator = cont.iteritems() + elif callable_attr(cont, 'items'): + iterator = iter(cont.items()) + return iterator + + def get(self, key, default=None): + if key in self: + return self._map[key][0].value + return default + + def getlist(self, key, default=[]): + """ + Returns: The list of values for if is in the dictionary, + else . If is not provided, an empty list is + returned. + """ + if key in self: + return [node.value for node in self._map[key]] + return default + + def setdefault(self, key, default=None): + if key in self: + return self[key] + self.add(key, default) + return default + + def setdefaultlist(self, key, defaultlist=[None]): + """ + Similar to setdefault() except is a list of values to set + for . If already exists, its existing list of values is + returned. + + If isn't a key and is an empty list, [], no values + are added for and will not be added as a key. + + Returns: List of 's values if exists in the dictionary, + otherwise . + """ + if key in self: + return self.getlist(key) + self.addlist(key, defaultlist) + return defaultlist + + def add(self, key, value=None): + """ + Add to the list of values for . If is not in the + dictionary, then is added as the sole value for . + + Example: + omd = omdict() + omd.add(1, 1) # omd.allitems() == [(1,1)] + omd.add(1, 11) # omd.allitems() == [(1,1), (1,11)] + omd.add(2, 2) # omd.allitems() == [(1,1), (1,11), (2,2)] + + Returns: . + """ + self._map.setdefault(key, []) + node = self._items.append(key, value) + self._map[key].append(node) + return self + + def addlist(self, key, valuelist=[]): + """ + Add the values in to the list of values for . If + is not in the dictionary, the values in become the values + for . + + Example: + omd = omdict([(1,1)]) + omd.addlist(1, [11, 111]) + omd.allitems() == [(1, 1), (1, 11), (1, 111)] + omd.addlist(2, [2]) + omd.allitems() == [(1, 1), (1, 11), (1, 111), (2, 2)] + + Returns: . + """ + for value in valuelist: + self.add(key, value) + return self + + def set(self, key, value=None): + """ + Sets 's value to . Identical in function to __setitem__(). + + Returns: . + """ + self[key] = value + return self + + def setlist(self, key, values): + """ + Sets 's list of values to . Existing items with key + are first replaced with new values from . Any remaining old + items that haven't been replaced with new values are deleted, and any + new values from that don't have corresponding items with + to replace are appended to the end of the list of all items. + + If values is an empty list, [], is deleted, equivalent in action + to del self[]. + + Example: + omd = omdict([(1,1), (2,2)]) + omd.setlist(1, [11, 111]) + omd.allitems() == [(1,11), (2,2), (1,111)] + + omd = omdict([(1,1), (1,11), (2,2), (1,111)]) + omd.setlist(1, [None]) + omd.allitems() == [(1,None), (2,2)] + + omd = omdict([(1,1), (1,11), (2,2), (1,111)]) + omd.setlist(1, []) + omd.allitems() == [(2,2)] + + Returns: . + """ + if not values and key in self: + self.pop(key) + else: + it = zip_longest( + list(self._map.get(key, [])), values, fillvalue=_absent) + for node, value in it: + if node is not _absent and value is not _absent: + node.value = value + elif node is _absent: + self.add(key, value) + elif value is _absent: + self._map[key].remove(node) + self._items.removenode(node) + return self + + def removevalues(self, key, values): + """ + Removes all from the values of . If has no + remaining values after removevalues(), the key is popped. + + Example: + omd = omdict([(1, 1), (1, 11), (1, 1), (1, 111)]) + omd.removevalues(1, [1, 111]) + omd.allitems() == [(1, 11)] + + Returns: . + """ + self.setlist(key, [v for v in self.getlist(key) if v not in values]) + return self + + def pop(self, key, default=_absent): + if key in self: + return self.poplist(key)[0] + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def poplist(self, key, default=_absent): + """ + If is in the dictionary, pop it and return its list of values. If + is not in the dictionary, return . KeyError is raised if + is not provided and is not in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.poplist(1) == [1, 11, 111] + omd.allitems() == [(2,2), (3,3)] + omd.poplist(2) == [2] + omd.allitems() == [(3,3)] + + Raises: KeyError if isn't in the dictionary and isn't + provided. + Returns: List of 's values. + """ + if key in self: + values = self.getlist(key) + del self._map[key] + for node, nodekey, nodevalue in self._items: + if nodekey == key: + self._items.removenode(node) + return values + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def popvalue(self, key, value=_absent, default=_absent, last=True): + """ + If is provided, pops the first or last (key,value) item in the + dictionary if is in the dictionary. + + If is not provided, pops the first or last value for if + is in the dictionary. + + If no longer has any values after a popvalue() call, is + removed from the dictionary. If isn't in the dictionary and + was provided, return default. KeyError is raised if + is not provided and is not in the dictionary. ValueError is + raised if is provided but isn't a value for . + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3), (2,22)]) + omd.popvalue(1) == 111 + omd.allitems() == [(1,11), (1,111), (2,2), (3,3), (2,22)] + omd.popvalue(1, last=False) == 1 + omd.allitems() == [(1,11), (2,2), (3,3), (2,22)] + omd.popvalue(2, 2) == 2 + omd.allitems() == [(1,11), (3,3), (2,22)] + omd.popvalue(1, 11) == 11 + omd.allitems() == [(3,3), (2,22)] + omd.popvalue('not a key', default='sup') == 'sup' + + Params: + last: Boolean whether to return 's first value ( is False) + or last value ( is True). + Raises: + KeyError if isn't in the dictionary and isn't + provided. + ValueError if isn't a value for . + Returns: The first or last of 's values. + """ + def pop_node_with_index(key, index): + node = self._map[key].pop(index) + if not self._map[key]: + del self._map[key] + self._items.removenode(node) + return node + + if key in self: + if value is not _absent: + if last: + pos = self.values(key)[::-1].index(value) + else: + pos = self.values(key).index(value) + if pos == -1: + raise ValueError(value) + else: + index = (len(self.values(key)) - 1 - pos) if last else pos + return pop_node_with_index(key, index).value + else: + return pop_node_with_index(key, -1 if last else 0).value + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def popitem(self, fromall=False, last=True): + """ + Pop and return a key:value item. + + If is False, items()[0] is popped if is False or + items()[-1] is popped if is True. All remaining items with the + same key are removed. + + If is True, allitems()[0] is popped if is False or + allitems()[-1] is popped if is True. Any remaining items with + the same key remain. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.popitem() == (3,3) + omd.popitem(fromall=False, last=False) == (1,1) + omd.popitem(fromall=False, last=False) == (2,2) + + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.popitem(fromall=True, last=False) == (1,1) + omd.popitem(fromall=True, last=False) == (1,11) + omd.popitem(fromall=True, last=True) == (3,3) + omd.popitem(fromall=True, last=False) == (1,111) + + Params: + fromall: Whether to pop an item from items() ( is True) or + allitems() ( is False). + last: Boolean whether to pop the first item or last item of items() + or allitems(). + Raises: KeyError if the dictionary is empty. + Returns: The first or last item from item() or allitem(). + """ + if not self._items: + raise KeyError('popitem(): %s is empty' % self.__class__.__name__) + + if fromall: + node = self._items[-1 if last else 0] + key = node.key + return key, self.popvalue(key, last=last) + else: + key = list(self._map.keys())[-1 if last else 0] + return key, self.pop(key) + + def poplistitem(self, last=True): + """ + Pop and return a key:valuelist item comprised of a key and that key's + list of values. If is False, a key:valuelist item comprised of + keys()[0] and its list of values is popped and returned. If is + True, a key:valuelist item comprised of keys()[-1] and its list of + values is popped and returned. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.poplistitem(last=True) == (3,[3]) + omd.poplistitem(last=False) == (1,[1,11,111]) + + Params: + last: Boolean whether to pop the first or last key and its associated + list of values. + Raises: KeyError if the dictionary is empty. + Returns: A two-tuple comprised of the first or last key and its + associated list of values. + """ + if not self._items: + s = 'poplistitem(): %s is empty' % self.__class__.__name__ + raise KeyError(s) + + key = self.keys()[-1 if last else 0] + return key, self.poplist(key) + + def items(self, key=_absent): + """ + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from iteritems(). Only items with key + are returned if is provided and is a dictionary key. + """ + return list(self.iteritems(key)) + + def keys(self): + return list(self.iterkeys()) + + def values(self, key=_absent): + """ + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from itervalues().If is provided and + is a dictionary key, only values of items with key are + returned. + """ + if key is not _absent and key in self._map: + return self.getlist(key) + return list(self.itervalues()) + + def lists(self): + """ + Returns: List created from iterlists(). + """ + return list(self.iterlists()) + + def listitems(self): + """ + Returns: List created from iterlistitems(). + """ + return list(self.iterlistitems()) + + def iteritems(self, key=_absent): + """ + Parity with dict.iteritems() except the optional parameter has + been added. If is provided, only items with the provided key are + iterated over. KeyError is raised if is provided and not in the + dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iteritems(1) -> (1,1) -> (1,11) -> (1,111) + omd.iteritems() -> (1,1) -> (2,2) -> (3,3) + + Raises: KeyError if is provided and not in the dictionary. + Returns: An iterator over the items() of the dictionary, or only items + with the key if is provided. + """ + if key is not _absent: + if key in self: + items = [(node.key, node.value) for node in self._map[key]] + return iter(items) + raise KeyError(key) + items = six.iteritems(self._map) + return iter((key, nodes[0].value) for (key, nodes) in items) + + def iterkeys(self): + return six.iterkeys(self._map) + + def itervalues(self, key=_absent): + """ + Parity with dict.itervalues() except the optional parameter has + been added. If is provided, only values from items with the + provided key are iterated over. KeyError is raised if is provided + and not in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.itervalues(1) -> 1 -> 11 -> 111 + omd.itervalues() -> 1 -> 11 -> 111 -> 2 -> 3 + + Raises: KeyError if is provided and isn't in the dictionary. + Returns: An iterator over the values() of the dictionary, or only the + values of key if is provided. + """ + if key is not _absent: + if key in self: + return iter([node.value for node in self._map[key]]) + raise KeyError(key) + return iter([nodes[0].value for nodes in six.itervalues(self._map)]) + + def allitems(self, key=_absent): + ''' + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from iterallitems(). + ''' + return list(self.iterallitems(key)) + + def allkeys(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.allkeys() == [1,1,1,2,3] + + Returns: List created from iterallkeys(). + ''' + return list(self.iterallkeys()) + + def allvalues(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.allvalues() == [1,11,111,2,3] + omd.allvalues(1) == [1,11,111] + + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from iterallvalues(). + ''' + return list(self.iterallvalues(key)) + + def iterallitems(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallitems() == (1,1) -> (1,11) -> (1,111) -> (2,2) -> (3,3) + omd.iterallitems(1) == (1,1) -> (1,11) -> (1,111) + + Raises: KeyError if is provided and not in the dictionary. + Returns: An iterator over every item in the diciontary. If is + provided, only items with the key are iterated over. + ''' + if key is not _absent: + # Raises KeyError if is not in self._map. + return self.iteritems(key) + return self._items.iteritems() + + def iterallkeys(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallkeys() == 1 -> 1 -> 1 -> 2 -> 3 + + Returns: An iterator over the keys of every item in the dictionary. + ''' + return self._items.iterkeys() + + def iterallvalues(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallvalues() == 1 -> 11 -> 111 -> 2 -> 3 + + Returns: An iterator over the values of every item in the dictionary. + ''' + if key is not _absent: + if key in self: + return iter(self.getlist(key)) + raise KeyError(key) + return self._items.itervalues() + + def iterlists(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterlists() -> [1,11,111] -> [2] -> [3] + + Returns: An iterator over the list comprised of the lists of values for + each key. + ''' + return map(lambda key: self.getlist(key), self) + + def iterlistitems(self): + """ + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterlistitems() -> (1,[1,11,111]) -> (2,[2]) -> (3,[3]) + + Returns: An iterator over the list of key:valuelist items. + """ + return map(lambda key: (key, self.getlist(key)), self) + + def reverse(self): + """ + Reverse the order of all items in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.reverse() + omd.allitems() == [(3,3), (2,2), (1,111), (1,11), (1,1)] + + Returns: . + """ + for key in six.iterkeys(self._map): + self._map[key].reverse() + self._items.reverse() + return self + + def __eq__(self, other): + if callable_attr(other, 'iterallitems'): + myiter, otheriter = self.iterallitems(), other.iterallitems() + for i1, i2 in zip_longest(myiter, otheriter, fillvalue=_absent): + if i1 != i2 or i1 is _absent or i2 is _absent: + return False + elif not hasattr(other, '__len__') or not hasattr(other, items_attr): + return False + # Ignore order so we can compare ordered omdicts with unordered dicts. + else: + if len(self) != len(other): + return False + for key, value in six.iteritems(other): + if self.get(key, _absent) != value: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __len__(self): + return len(self._map) + + def __iter__(self): + for key in self.iterkeys(): + yield key + + def __contains__(self, key): + return key in self._map + + def __getitem__(self, key): + if key in self: + return self.get(key) + raise KeyError(key) + + def __setitem__(self, key, value): + self.setlist(key, [value]) + + def __delitem__(self, key): + return self.pop(key) + + def __nonzero__(self): + return bool(self._map) + + def __str__(self): + return '{%s}' % ', '.join( + map(lambda p: '%r: %r' % (p[0], p[1]), self.iterallitems())) + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, self.allitems()) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index f4232b63..deba443d 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -51,3 +51,4 @@ resolvelib==0.2.2 backports.functools_lru_cache==1.5 pep517==0.5.0 pytoml==0.1.20 +orderedmultidict==1.0 From cebe0ac54086737d6b6093d264c76e655382a214 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 4 Mar 2019 12:32:30 -0500 Subject: [PATCH 13/21] Update vnedored dependency list Signed-off-by: Dan Ryan --- news/3298.vendor.rst | 9 +++++++-- pipenv/vendor/vendor.txt | 18 ++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/news/3298.vendor.rst b/news/3298.vendor.rst index f829fcf0..79fb60b9 100644 --- a/news/3298.vendor.rst +++ b/news/3298.vendor.rst @@ -1,23 +1,28 @@ - Updated vendored dependencies: + - **attrs**: ``18.2.0`` => ``19.1.0`` - **certifi**: ``2018.10.15`` => ``2018.11.29`` - **cached_property**: ``1.4.3`` => ``1.5.1`` - **colorama**: ``0.3.9`` => ``0.4.1`` - **idna**: ``2.7`` => ``2.8`` + - **markupsafe**: ``1.0`` => ``1.1.1`` + - **orderedmultidict**: ``(new)`` => ``1.0`` - **packaging**: ``18.0`` => ``19.0`` + - **parse**: ``1.9.0`` => ``1.11.1`` - **pathlib2**: ``2.3.2`` => ``2.3.3`` - **pep517**: ``(new)`` => ``0.5.0`` - - **pipdeptree**: ``0.13.0`` => ``0.13.1`` + - **pipdeptree**: ``0.13.0`` => ``0.13.2`` - **pyparsing**: ``2.2.2`` => ``2.3.1`` - **python-dotenv**: ``0.9.1`` => ``0.10.1`` - **pythonfinder**: ``1.1.10`` => ``1.2.0`` - **pytoml**: ``(new)`` => ``0.1.20`` - **requests**: ``2.20.1`` => ``2.21.0`` - - **requirementslib**: ``1.3.3`` => ``1.4.0`` + - **requirementslib**: ``1.3.3`` => ``1.4.2`` - **shellingham**: ``1.2.7`` => ``1.2.8`` - **six**: ``1.11.0`` => ``1.12.0`` - **tomlkit**: ``0.5.2`` => ``0.5.3`` - **urllib3**: ``1.24`` => ``1.24.1`` - **vistir**: ``0.3.0`` => ``0.3.1`` + - **yaspin**: ``0.14.0`` => ``0.14.1`` Removed vendored dependency **cursor**. diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index deba443d..39017973 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -13,22 +13,22 @@ python-dotenv==0.10.1 first==2.0.1 iso8601==0.1.12 jinja2==2.10 -markupsafe==1.0 -parse==1.9.0 +markupsafe==1.1.1 +parse==1.11.1 pathlib2==2.3.3 scandir==1.9 -pipdeptree==0.13.1 +pipdeptree==0.13.2 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.10 +pythonfinder==1.2.0 requests==2.21.0 chardet==3.0.4 idna==2.8 urllib3==1.24.1 certifi==2018.11.29 -requirementslib==1.4.0 - attrs==18.2.0 +requirementslib==1.4.2 + attrs==19.1.0 distlib==0.2.8 packaging==19.0 pyparsing==2.3.1 @@ -40,13 +40,11 @@ semver==2.8.1 shutilwhich==1.1.0 toml==0.10.0 cached-property==1.5.1 -vistir==0.3.0 +vistir==0.3.1 pip-shims==0.3.2 enum34==1.1.6 -yaspin==0.14.0 +yaspin==0.14.1 cerberus==1.2 -git+https://github.com/sarugaku/passa.git@master#egg=passa -cursor==1.2.0 resolvelib==0.2.2 backports.functools_lru_cache==1.5 pep517==0.5.0 From 37916cb7615dca1ee8697c54fe996c8255b9769a Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 4 Mar 2019 12:54:07 -0500 Subject: [PATCH 14/21] Re-add passa Signed-off-by: Dan Ryan --- pipenv/vendor/vendor.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 39017973..f1fe99c0 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -49,4 +49,5 @@ resolvelib==0.2.2 backports.functools_lru_cache==1.5 pep517==0.5.0 pytoml==0.1.20 +git+https://github.com/sarugaku/passa.git@master#egg=passa orderedmultidict==1.0 From 716dfb48044b50892383ffb73e19554972e250e3 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 4 Mar 2019 13:35:32 -0500 Subject: [PATCH 15/21] Update vistir patch Signed-off-by: Dan Ryan --- tasks/vendoring/patches/vendor/vistir-imports.patch | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tasks/vendoring/patches/vendor/vistir-imports.patch b/tasks/vendoring/patches/vendor/vistir-imports.patch index 544b5564..d1dc4e3a 100644 --- a/tasks/vendoring/patches/vendor/vistir-imports.patch +++ b/tasks/vendoring/patches/vendor/vistir-imports.patch @@ -15,21 +15,23 @@ diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index 9ae33fdc..ec3b65cb 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py -@@ -43,11 +43,11 @@ if sys.version_info >= (3, 5): +@@ -43,12 +43,12 @@ if sys.version_info >= (3, 5): from functools import lru_cache else: - from pathlib2 import Path +- from pathlib2 import Path - from backports.functools_lru_cache import lru_cache ++ from pipenv.vendor.pathlib2 import Path + from pipenv.vendor.backports.functools_lru_cache import lru_cache - from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile + if sys.version_info < (3, 3): - from backports.shutil_get_terminal_size import get_terminal_size + from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + NamedTemporaryFile = _NamedTemporaryFile else: from tempfile import NamedTemporaryFile -@@ -56,7 +56,7 @@ else: +@@ -57,7 +57,7 @@ else: try: from weakref import finalize except ImportError: From 2d1b4bd981561f11c25fc9a9a3d015952c6368aa Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 4 Mar 2019 13:36:08 -0500 Subject: [PATCH 16/21] Update setup dependencies Signed-off-by: Dan Ryan --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 994acf96..3fb48591 100644 --- a/setup.py +++ b/setup.py @@ -128,10 +128,11 @@ setup( ], }, python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*", - setup_requires=["invoke", "parver"], + setup_requires=["invoke", "parver", ], install_requires=required, extras_require={ - "test": ["pytest<4.0", "pytest-tap", "pytest-xdist", "flaky", "mock"] + "test": ["pytest<4.0", "pytest-tap", "pytest-xdist", "flaky", "mock"], + "dev": ["towncrier", "bs4"], }, include_package_data=True, license="MIT", From dc41e66df686661e20c4b6a56d8e67f9acc6bafc Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 4 Mar 2019 14:33:39 -0500 Subject: [PATCH 17/21] Re-run vendoring script Signed-off-by: Dan Ryan --- pipenv/patched/safety/__init__.py | 2 +- pipenv/patched/safety/cli.py | 94 +++- pipenv/patched/safety/formatter.py | 8 +- pipenv/patched/safety/safety.py | 31 +- pipenv/patched/safety/util.py | 9 +- pipenv/vendor/attr/__init__.py | 2 +- pipenv/vendor/attr/__init__.pyi | 27 +- pipenv/vendor/attr/_compat.py | 46 +- pipenv/vendor/attr/_make.py | 106 +++- pipenv/vendor/attr/filters.pyi | 6 +- pipenv/vendor/attr/validators.py | 114 +++- pipenv/vendor/attr/validators.pyi | 10 + pipenv/vendor/cursor/LICENSE | 5 - pipenv/vendor/cursor/__init__.py | 4 - pipenv/vendor/cursor/cursor.py | 57 -- pipenv/vendor/markupsafe/LICENSE | 33 -- pipenv/vendor/markupsafe/LICENSE.rst | 28 + pipenv/vendor/markupsafe/__init__.py | 258 +++++---- pipenv/vendor/markupsafe/_compat.py | 23 +- pipenv/vendor/markupsafe/_constants.py | 517 +++++++++--------- pipenv/vendor/markupsafe/_native.py | 67 ++- pipenv/vendor/markupsafe/_speedups.c | 214 +++++++- .../orderedmultidict/{LICENSE => LICENSE.md} | 2 +- pipenv/vendor/parse.py | 131 +++-- pipenv/vendor/pipdeptree.py | 13 +- pipenv/vendor/pythonfinder/__init__.py | 13 +- pipenv/vendor/pythonfinder/cli.py | 4 +- pipenv/vendor/pythonfinder/environment.py | 3 + pipenv/vendor/pythonfinder/models/mixins.py | 40 +- pipenv/vendor/pythonfinder/models/path.py | 136 +++-- pipenv/vendor/pythonfinder/models/python.py | 102 ++-- pipenv/vendor/pythonfinder/models/windows.py | 49 +- pipenv/vendor/pythonfinder/pythonfinder.py | 49 +- pipenv/vendor/pythonfinder/utils.py | 95 ++-- pipenv/vendor/pytoml/LICENSE | 16 + pipenv/vendor/requirementslib/__init__.py | 12 +- .../requirementslib/models/dependencies.py | 18 +- pipenv/vendor/requirementslib/utils.py | 2 +- pipenv/vendor/vistir/__init__.py | 29 +- pipenv/vendor/vistir/backports/__init__.py | 6 +- pipenv/vendor/vistir/backports/functools.py | 24 +- pipenv/vendor/vistir/backports/tempfile.py | 13 +- pipenv/vendor/vistir/cmdparse.py | 11 +- pipenv/vendor/vistir/compat.py | 41 +- pipenv/vendor/vistir/contextmanagers.py | 29 +- pipenv/vendor/vistir/cursor.py | 7 +- pipenv/vendor/vistir/misc.py | 168 +++--- pipenv/vendor/vistir/path.py | 42 +- pipenv/vendor/vistir/spin.py | 25 +- pipenv/vendor/vistir/termcolors.py | 84 ++- pipenv/vendor/yaspin/__version__.py | 2 +- pipenv/vendor/yaspin/api.py | 6 +- 52 files changed, 1721 insertions(+), 1112 deletions(-) delete mode 100644 pipenv/vendor/cursor/LICENSE delete mode 100644 pipenv/vendor/cursor/__init__.py delete mode 100644 pipenv/vendor/cursor/cursor.py delete mode 100644 pipenv/vendor/markupsafe/LICENSE create mode 100644 pipenv/vendor/markupsafe/LICENSE.rst rename pipenv/vendor/orderedmultidict/{LICENSE => LICENSE.md} (95%) mode change 100755 => 100644 create mode 100644 pipenv/vendor/pytoml/LICENSE diff --git a/pipenv/patched/safety/__init__.py b/pipenv/patched/safety/__init__.py index 56b497d2..69563274 100644 --- a/pipenv/patched/safety/__init__.py +++ b/pipenv/patched/safety/__init__.py @@ -2,4 +2,4 @@ __author__ = """pyup.io""" __email__ = 'support@pyup.io' -__version__ = '1.8.4' +__version__ = '1.8.5' diff --git a/pipenv/patched/safety/cli.py b/pipenv/patched/safety/cli.py index 9ebd9e8a..2e8f88df 100644 --- a/pipenv/patched/safety/cli.py +++ b/pipenv/patched/safety/cli.py @@ -6,17 +6,13 @@ from safety import __version__ from safety import safety from safety.formatter import report import itertools -from safety.util import read_requirements +from safety.util import read_requirements, read_vulnerabilities from safety.errors import DatabaseFetchError, DatabaseFileNotFoundError, InvalidKeyError - try: - # pip 9 - from pipenv.patched.notpip import get_installed_distributions + from json.decoder import JSONDecodeError except ImportError: - # pip 10 - from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions - + JSONDecodeError = ValueError @click.group() @click.version_option(version=__version__) @@ -46,10 +42,17 @@ def cli(): help="Read input from one (or multiple) requirement files. Default: empty") @click.option("ignore", "--ignore", "-i", multiple=True, type=str, default=[], help="Ignore one (or multiple) vulnerabilities by ID. Default: empty") -def check(key, db, json, full_report, bare, stdin, files, cache, ignore): - +@click.option("--output", "-o", default="", + help="Path to where output file will be placed. Default: empty") +@click.option("proxyhost", "--proxy-host", "-ph", multiple=False, type=str, default=None, + help="Proxy host IP or DNS --proxy-host") +@click.option("proxyport", "--proxy-port", "-pp", multiple=False, type=int, default=80, + help="Proxy port number --proxy-port") +@click.option("proxyprotocol", "--proxy-protocol", "-pr", multiple=False, type=str, default='http', + help="Proxy protocol (https or http) --proxy-protocol") +def check(key, db, json, full_report, bare, stdin, files, cache, ignore, output, proxyprotocol, proxyhost, proxyport): if files and stdin: - click.secho("Can't read from --stdin and --file at the same time, exiting", fg="red") + click.secho("Can't read from --stdin and --file at the same time, exiting", fg="red", file=sys.stderr) sys.exit(-1) if files: @@ -57,33 +60,72 @@ def check(key, db, json, full_report, bare, stdin, files, cache, ignore): elif stdin: packages = list(read_requirements(sys.stdin)) else: - packages = get_installed_distributions() - + import pkg_resources + packages = [ + d for d in pkg_resources.working_set + if d.key not in {"python", "wsgiref", "argparse"} + ] + proxy_dictionary = {} + if proxyhost is not None: + if proxyprotocol in ["http", "https"]: + proxy_dictionary = {proxyprotocol: "{0}://{1}:{2}".format(proxyprotocol, proxyhost, str(proxyport))} + else: + click.secho("Proxy Protocol should be http or https only.", fg="red") + sys.exit(-1) try: - vulns = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_ids=ignore) - click.secho(report( - vulns=vulns, - full=full_report, - json_report=json, - bare_report=bare, - checked_packages=len(packages), - db=db, - key=key - ) - ) + vulns = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_ids=ignore, proxy=proxy_dictionary) + output_report = report(vulns=vulns, + full=full_report, + json_report=json, + bare_report=bare, + checked_packages=len(packages), + db=db, + key=key) + + if output: + with open(output, 'w+') as output_file: + output_file.write(output_report) + else: + click.secho(output_report, nl=False if bare and not vulns else True) sys.exit(-1 if vulns else 0) except InvalidKeyError: click.secho("Your API Key '{key}' is invalid. See {link}".format( key=key, link='https://goo.gl/O7Y1rS'), - fg="red") + fg="red", + file=sys.stderr) sys.exit(-1) except DatabaseFileNotFoundError: - click.secho("Unable to load vulnerability database from {db}".format(db=db), fg="red") + click.secho("Unable to load vulnerability database from {db}".format(db=db), fg="red", file=sys.stderr) sys.exit(-1) except DatabaseFetchError: - click.secho("Unable to load vulnerability database", fg="red") + click.secho("Unable to load vulnerability database", fg="red", file=sys.stderr) sys.exit(-1) +@cli.command() +@click.option("--full-report/--short-report", default=False, + help='Full reports include a security advisory (if available). Default: ' + '--short-report') +@click.option("--bare/--not-bare", default=False, + help='Output vulnerable packages only. Useful in combination with other tools.' + 'Default: --not-bare') +@click.option("file", "--file", "-f", type=click.File(), required=True, + help="Read input from an insecure report file. Default: empty") +def review(full_report, bare, file): + if full_report and bare: + click.secho("Can't choose both --bare and --full-report/--short-report", fg="red") + sys.exit(-1) + + try: + input_vulns = read_vulnerabilities(file) + except JSONDecodeError: + click.secho("Not a valid JSON file", fg="red") + sys.exit(-1) + + vulns = safety.review(input_vulns) + output_report = report(vulns=vulns, full=full_report, bare_report=bare) + click.secho(output_report, nl=False if bare and not vulns else True) + + if __name__ == "__main__": cli() diff --git a/pipenv/patched/safety/formatter.py b/pipenv/patched/safety/formatter.py index 8bc57ec1..c19bff1b 100644 --- a/pipenv/patched/safety/formatter.py +++ b/pipenv/patched/safety/formatter.py @@ -3,6 +3,7 @@ import platform import sys import json import os +import textwrap # python 2.7 compat try: @@ -110,9 +111,10 @@ class SheetReport(object): descr = get_advisory(vuln) - for chunk in [descr[i:i + 76] for i in range(0, len(descr), 76)]: - - for line in chunk.splitlines(): + for pn, paragraph in enumerate(descr.replace('\r', '').split('\n\n')): + if pn: + table.append("│ {:76} │".format('')) + for line in textwrap.wrap(paragraph, width=76): try: table.append("│ {:76} │".format(line.encode('utf-8'))) except TypeError: diff --git a/pipenv/patched/safety/safety.py b/pipenv/patched/safety/safety.py index 99b99125..871bd775 100644 --- a/pipenv/patched/safety/safety.py +++ b/pipenv/patched/safety/safety.py @@ -9,6 +9,7 @@ import json import time import errno + class Vulnerability(namedtuple("Vulnerability", ["name", "spec", "version", "advisory", "vuln_id"])): pass @@ -64,7 +65,7 @@ def write_to_cache(db_name, data): f.write(json.dumps(cache)) -def fetch_database_url(mirror, db_name, key, cached): +def fetch_database_url(mirror, db_name, key, cached, proxy): headers = {} if key: @@ -74,9 +75,8 @@ def fetch_database_url(mirror, db_name, key, cached): cached_data = get_from_cache(db_name=db_name) if cached_data: return cached_data - url = mirror + db_name - r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers) + r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy) if r.status_code == 200: data = r.json() if cached: @@ -94,7 +94,7 @@ def fetch_database_file(path, db_name): return json.loads(f.read()) -def fetch_database(full=False, key=False, db=False, cached=False): +def fetch_database(full=False, key=False, db=False, cached=False, proxy={}): if db: mirrors = [db] @@ -105,7 +105,7 @@ def fetch_database(full=False, key=False, db=False, cached=False): for mirror in mirrors: # mirror can either be a local path or a URL if mirror.startswith("http://") or mirror.startswith("https://"): - data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached) + data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy) else: data = fetch_database_file(mirror, db_name=db_name) if data: @@ -120,10 +120,9 @@ def get_vulnerabilities(pkg, spec, db): yield entry -def check(packages, key, db_mirror, cached, ignore_ids): - +def check(packages, key, db_mirror, cached, ignore_ids, proxy): key = key if key else os.environ.get("SAFETY_API_KEY", False) - db = fetch_database(key=key, db=db_mirror, cached=cached) + db = fetch_database(key=key, db=db_mirror, cached=cached, proxy=proxy) db_full = None vulnerable_packages = frozenset(db.keys()) vulnerable = [] @@ -152,3 +151,19 @@ def check(packages, key, db_mirror, cached, ignore_ids): ) ) return vulnerable + + +def review(vulnerabilities): + vulnerable = [] + for vuln in vulnerabilities: + current_vuln = { + "name": vuln[0], + "spec": vuln[1], + "version": vuln[2], + "advisory": vuln[3], + "vuln_id": vuln[4], + } + vulnerable.append( + Vulnerability(**current_vuln) + ) + return vulnerable diff --git a/pipenv/patched/safety/util.py b/pipenv/patched/safety/util.py index 2e6efaec..16062f41 100644 --- a/pipenv/patched/safety/util.py +++ b/pipenv/patched/safety/util.py @@ -1,11 +1,17 @@ from dparse.parser import setuptools_parse_requirements_backport as _parse_requirements from collections import namedtuple import click +import sys +import json import os Package = namedtuple("Package", ["key", "version"]) RequirementFile = namedtuple("RequirementFile", ["path"]) +def read_vulnerabilities(fh): + return json.load(fh) + + def iter_lines(fh, lineno=0): for line in fh.readlines()[lineno:]: yield line @@ -85,7 +91,8 @@ def read_requirements(fh, resolve=False): "Warning: unpinned requirement '{req}' found in {fname}, " "unable to check.".format(req=req.name, fname=fname), - fg="yellow" + fg="yellow", + file=sys.stderr ) except ValueError: continue diff --git a/pipenv/vendor/attr/__init__.py b/pipenv/vendor/attr/__init__.py index debfd57b..0ebe5197 100644 --- a/pipenv/vendor/attr/__init__.py +++ b/pipenv/vendor/attr/__init__.py @@ -18,7 +18,7 @@ from ._make import ( ) -__version__ = "18.2.0" +__version__ = "19.1.0" __title__ = "attrs" __description__ = "Classes Without Boilerplate" diff --git a/pipenv/vendor/attr/__init__.pyi b/pipenv/vendor/attr/__init__.pyi index 492fb85e..fcb93b18 100644 --- a/pipenv/vendor/attr/__init__.pyi +++ b/pipenv/vendor/attr/__init__.pyi @@ -23,9 +23,9 @@ from . import validators as validators _T = TypeVar("_T") _C = TypeVar("_C", bound=type) -_ValidatorType = Callable[[Any, Attribute, _T], Any] +_ValidatorType = Callable[[Any, Attribute[_T], _T], Any] _ConverterType = Callable[[Any], _T] -_FilterType = Callable[[Attribute, Any], bool] +_FilterType = Callable[[Attribute[_T], _T], bool] # FIXME: in reality, if multiple validators are passed they must be in a list or tuple, # but those are invariant and so would prevent subtypes of _ValidatorType from working # when passed in a list or tuple. @@ -57,10 +57,10 @@ class Attribute(Generic[_T]): metadata: Dict[Any, Any] type: Optional[Type[_T]] kw_only: bool - def __lt__(self, x: Attribute) -> bool: ... - def __le__(self, x: Attribute) -> bool: ... - def __gt__(self, x: Attribute) -> bool: ... - def __ge__(self, x: Attribute) -> bool: ... + def __lt__(self, x: Attribute[_T]) -> bool: ... + def __le__(self, x: Attribute[_T]) -> bool: ... + def __gt__(self, x: Attribute[_T]) -> bool: ... + def __ge__(self, x: Attribute[_T]) -> bool: ... # NOTE: We had several choices for the annotation to use for type arg: # 1) Type[_T] @@ -167,6 +167,7 @@ def attrs( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., ) -> _C: ... @overload def attrs( @@ -184,14 +185,15 @@ def attrs( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., ) -> Callable[[_C], _C]: ... # TODO: add support for returning NamedTuple from the mypy plugin -class _Fields(Tuple[Attribute, ...]): - def __getattr__(self, name: str) -> Attribute: ... +class _Fields(Tuple[Attribute[Any], ...]): + def __getattr__(self, name: str) -> Attribute[Any]: ... def fields(cls: type) -> _Fields: ... -def fields_dict(cls: type) -> Dict[str, Attribute]: ... +def fields_dict(cls: type) -> Dict[str, Attribute[Any]]: ... def validate(inst: Any) -> None: ... # TODO: add support for returning a proper attrs class from the mypy plugin @@ -212,6 +214,7 @@ def make_class( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., ) -> type: ... # _funcs -- @@ -223,7 +226,7 @@ def make_class( def asdict( inst: Any, recurse: bool = ..., - filter: Optional[_FilterType] = ..., + filter: Optional[_FilterType[Any]] = ..., dict_factory: Type[Mapping[Any, Any]] = ..., retain_collection_types: bool = ..., ) -> Dict[str, Any]: ... @@ -232,8 +235,8 @@ def asdict( def astuple( inst: Any, recurse: bool = ..., - filter: Optional[_FilterType] = ..., - tuple_factory: Type[Sequence] = ..., + filter: Optional[_FilterType[Any]] = ..., + tuple_factory: Type[Sequence[Any]] = ..., retain_collection_types: bool = ..., ) -> Tuple[Any, ...]: ... def has(cls: type) -> bool: ... diff --git a/pipenv/vendor/attr/_compat.py b/pipenv/vendor/attr/_compat.py index 5bb06593..9a99dcd9 100644 --- a/pipenv/vendor/attr/_compat.py +++ b/pipenv/vendor/attr/_compat.py @@ -20,6 +20,7 @@ else: if PY2: from UserDict import IterableUserDict + from collections import Mapping, Sequence # noqa # We 'bundle' isclass instead of using inspect as importing inspect is # fairly expensive (order of 10-15 ms for a modern machine in 2016) @@ -89,8 +90,27 @@ if PY2: res.data.update(d) # We blocked update, so we have to do it like this. return res + def just_warn(*args, **kw): # pragma: nocover + """ + We only warn on Python 3 because we are not aware of any concrete + consequences of not setting the cell on Python 2. + """ -else: + +else: # Python 3 and later. + from collections.abc import Mapping, Sequence # noqa + + def just_warn(*args, **kw): + """ + We only warn on Python 3 because we are not aware of any concrete + consequences of not setting the cell on Python 2. + """ + warnings.warn( + "Missing ctypes. Some features like bare super() or accessing " + "__class__ will not work with slotted classes.", + RuntimeWarning, + stacklevel=2, + ) def isclass(klass): return isinstance(klass, type) @@ -113,30 +133,6 @@ def import_ctypes(): return ctypes -if not PY2: - - def just_warn(*args, **kw): - """ - We only warn on Python 3 because we are not aware of any concrete - consequences of not setting the cell on Python 2. - """ - warnings.warn( - "Missing ctypes. Some features like bare super() or accessing " - "__class__ will not work with slots classes.", - RuntimeWarning, - stacklevel=2, - ) - - -else: - - def just_warn(*args, **kw): # pragma: nocover - """ - We only warn on Python 3 because we are not aware of any concrete - consequences of not setting the cell on Python 2. - """ - - def make_set_closure_cell(): """ Moved into a function for testability. diff --git a/pipenv/vendor/attr/_make.py b/pipenv/vendor/attr/_make.py index f7fd05e7..827175a4 100644 --- a/pipenv/vendor/attr/_make.py +++ b/pipenv/vendor/attr/_make.py @@ -409,12 +409,11 @@ def _transform_attrs(cls, these, auto_attribs, kw_only): a.kw_only is False ): had_default = True - if was_kw_only is True and a.kw_only is False: + if was_kw_only is True and a.kw_only is False and a.init is True: raise ValueError( "Non keyword-only attributes are not allowed after a " - "keyword-only attribute. Attribute in question: {a!r}".format( - a=a - ) + "keyword-only attribute (unless they are init=False). " + "Attribute in question: {a!r}".format(a=a) ) if was_kw_only is False and a.init is True and a.kw_only is True: was_kw_only = True @@ -454,6 +453,7 @@ class _ClassBuilder(object): "_has_post_init", "_delete_attribs", "_base_attr_map", + "_is_exc", ) def __init__( @@ -466,6 +466,7 @@ class _ClassBuilder(object): auto_attribs, kw_only, cache_hash, + is_exc, ): attrs, base_attrs, base_map = _transform_attrs( cls, these, auto_attribs, kw_only @@ -483,6 +484,7 @@ class _ClassBuilder(object): self._cache_hash = cache_hash self._has_post_init = bool(getattr(cls, "__attrs_post_init__", False)) self._delete_attribs = not bool(these) + self._is_exc = is_exc self._cls_dict["__attrs_attrs__"] = self._attrs @@ -530,6 +532,26 @@ class _ClassBuilder(object): for name, value in self._cls_dict.items(): setattr(cls, name, value) + # Attach __setstate__. This is necessary to clear the hash code + # cache on deserialization. See issue + # https://github.com/python-attrs/attrs/issues/482 . + # Note that this code only handles setstate for dict classes. + # For slotted classes, see similar code in _create_slots_class . + if self._cache_hash: + existing_set_state_method = getattr(cls, "__setstate__", None) + if existing_set_state_method: + raise NotImplementedError( + "Currently you cannot use hash caching if " + "you specify your own __setstate__ method." + "See https://github.com/python-attrs/attrs/issues/494 ." + ) + + def cache_hash_set_state(chss_self, _): + # clear hash code cache + setattr(chss_self, _hash_cache_field, None) + + setattr(cls, "__setstate__", cache_hash_set_state) + return cls def _create_slots_class(self): @@ -582,6 +604,8 @@ class _ClassBuilder(object): """ return tuple(getattr(self, name) for name in state_attr_names) + hash_caching_enabled = self._cache_hash + def slots_setstate(self, state): """ Automatically created by attrs. @@ -589,6 +613,13 @@ class _ClassBuilder(object): __bound_setattr = _obj_setattr.__get__(self, Attribute) for name, value in zip(state_attr_names, state): __bound_setattr(name, value) + # Clearing the hash code cache on deserialization is needed + # because hash codes can change from run to run. See issue + # https://github.com/python-attrs/attrs/issues/482 . + # Note that this code only handles setstate for slotted classes. + # For dict classes, see similar code in _patch_original_class . + if hash_caching_enabled: + __bound_setattr(_hash_cache_field, None) # slots and frozen require __getstate__/__setstate__ to work cd["__getstate__"] = slots_getstate @@ -660,6 +691,7 @@ class _ClassBuilder(object): self._slots, self._cache_hash, self._base_attr_map, + self._is_exc, ) ) @@ -710,6 +742,7 @@ def attrs( auto_attribs=False, kw_only=False, cache_hash=False, + auto_exc=False, ): r""" A class decorator that adds `dunder @@ -815,10 +848,23 @@ def attrs( :param bool cache_hash: Ensure that the object's hash code is computed only once and stored on the object. If this is set to ``True``, hashing must be either explicitly or implicitly enabled for this - class. If the hash code is cached, then no attributes of this - class which participate in hash code computation may be mutated - after object creation. + class. If the hash code is cached, avoid any reassignments of + fields involved in hash code computation or mutations of the objects + those fields point to after object creation. If such changes occur, + the behavior of the object's hash code is undefined. + :param bool auto_exc: If the class subclasses :class:`BaseException` + (which implicitly includes any subclass of any exception), the + following happens to behave like a well-behaved Python exceptions + class: + - the values for *cmp* and *hash* are ignored and the instances compare + and hash by the instance's ids (N.B. ``attrs`` will *not* remove + existing implementations of ``__hash__`` or the equality methods. It + just won't add own ones.), + - all attributes that are either passed into ``__init__`` or have a + default value are additionally available as a tuple in the ``args`` + attribute, + - the value of *str* is ignored leaving ``__str__`` to base classes. .. versionadded:: 16.0.0 *slots* .. versionadded:: 16.1.0 *frozen* @@ -838,12 +884,16 @@ def attrs( to each other. .. versionadded:: 18.2.0 *kw_only* .. versionadded:: 18.2.0 *cache_hash* + .. versionadded:: 19.1.0 *auto_exc* """ def wrap(cls): + if getattr(cls, "__class__", None) is None: raise TypeError("attrs only works with new-style classes.") + is_exc = auto_exc is True and issubclass(cls, BaseException) + builder = _ClassBuilder( cls, these, @@ -853,13 +903,14 @@ def attrs( auto_attribs, kw_only, cache_hash, + is_exc, ) if repr is True: builder.add_repr(repr_ns) if str is True: builder.add_str() - if cmp is True: + if cmp is True and not is_exc: builder.add_cmp() if hash is not True and hash is not False and hash is not None: @@ -874,7 +925,11 @@ def attrs( " hashing must be either explicitly or implicitly " "enabled." ) - elif hash is True or (hash is None and cmp is True and frozen is True): + elif ( + hash is True + or (hash is None and cmp is True and frozen is True) + and is_exc is False + ): builder.add_hash() else: if cache_hash: @@ -1213,7 +1268,9 @@ def _add_repr(cls, ns=None, attrs=None): return cls -def _make_init(attrs, post_init, frozen, slots, cache_hash, base_attr_map): +def _make_init( + attrs, post_init, frozen, slots, cache_hash, base_attr_map, is_exc +): attrs = [a for a in attrs if a.init or a.default is not NOTHING] # We cache the generated init methods for the same kinds of attributes. @@ -1222,16 +1279,18 @@ def _make_init(attrs, post_init, frozen, slots, cache_hash, base_attr_map): unique_filename = "".format(sha1.hexdigest()) script, globs, annotations = _attrs_to_init_script( - attrs, frozen, slots, post_init, cache_hash, base_attr_map + attrs, frozen, slots, post_init, cache_hash, base_attr_map, is_exc ) locs = {} bytecode = compile(script, unique_filename, "exec") attr_dict = dict((a.name, a) for a in attrs) globs.update({"NOTHING": NOTHING, "attr_dict": attr_dict}) + if frozen is True: # Save the lookup overhead in __init__ if we need to circumvent # immutability. globs["_cached_setattr"] = _obj_setattr + eval(bytecode, globs, locs) # In order of debuggers like PDB being able to step through the code, @@ -1245,24 +1304,10 @@ def _make_init(attrs, post_init, frozen, slots, cache_hash, base_attr_map): __init__ = locs["__init__"] __init__.__annotations__ = annotations + return __init__ -def _add_init(cls, frozen): - """ - Add a __init__ method to *cls*. If *frozen* is True, make it immutable. - """ - cls.__init__ = _make_init( - cls.__attrs_attrs__, - getattr(cls, "__attrs_post_init__", False), - frozen, - _is_slot_cls(cls), - cache_hash=False, - base_attr_map={}, - ) - return cls - - def fields(cls): """ Return the tuple of ``attrs`` attributes for a class. @@ -1348,7 +1393,7 @@ def _is_slot_attr(a_name, base_attr_map): def _attrs_to_init_script( - attrs, frozen, slots, post_init, cache_hash, base_attr_map + attrs, frozen, slots, post_init, cache_hash, base_attr_map, is_exc ): """ Return a script of an initializer for *attrs* and a dict of globals. @@ -1597,6 +1642,13 @@ def _attrs_to_init_script( init_hash_cache = "self.%s = %s" lines.append(init_hash_cache % (_hash_cache_field, "None")) + # For exceptions we rely on BaseException.__init__ for proper + # initialization. + if is_exc: + vals = ",".join("self." + a.name for a in attrs if a.init) + + lines.append("BaseException.__init__(self, %s)" % (vals,)) + args = ", ".join(args) if kw_only_args: if PY2: diff --git a/pipenv/vendor/attr/filters.pyi b/pipenv/vendor/attr/filters.pyi index a618140c..68368fe2 100644 --- a/pipenv/vendor/attr/filters.pyi +++ b/pipenv/vendor/attr/filters.pyi @@ -1,5 +1,5 @@ -from typing import Union +from typing import Union, Any from . import Attribute, _FilterType -def include(*what: Union[type, Attribute]) -> _FilterType: ... -def exclude(*what: Union[type, Attribute]) -> _FilterType: ... +def include(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... +def exclude(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... diff --git a/pipenv/vendor/attr/validators.py b/pipenv/vendor/attr/validators.py index f12d0aa5..7fc4446b 100644 --- a/pipenv/vendor/attr/validators.py +++ b/pipenv/vendor/attr/validators.py @@ -136,7 +136,7 @@ class _InValidator(object): def __call__(self, inst, attr, value): try: in_options = value in self.options - except TypeError as e: # e.g. `1 in "abc"` + except TypeError: # e.g. `1 in "abc"` in_options = False if not in_options: @@ -168,3 +168,115 @@ def in_(options): .. versionadded:: 17.1.0 """ return _InValidator(options) + + +@attrs(repr=False, slots=False, hash=True) +class _IsCallableValidator(object): + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not callable(value): + raise TypeError("'{name}' must be callable".format(name=attr.name)) + + def __repr__(self): + return "" + + +def is_callable(): + """ + A validator that raises a :class:`TypeError` if the initializer is called + with a value for this particular attribute that is not callable. + + .. versionadded:: 19.1.0 + + :raises TypeError: With a human readable error message containing the + attribute (of type :class:`attr.Attribute`) name. + """ + return _IsCallableValidator() + + +@attrs(repr=False, slots=True, hash=True) +class _DeepIterable(object): + member_validator = attrib(validator=is_callable()) + iterable_validator = attrib( + default=None, validator=optional(is_callable()) + ) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.iterable_validator is not None: + self.iterable_validator(inst, attr, value) + + for member in value: + self.member_validator(inst, attr, member) + + def __repr__(self): + iterable_identifier = ( + "" + if self.iterable_validator is None + else " {iterable!r}".format(iterable=self.iterable_validator) + ) + return ( + "" + ).format( + iterable_identifier=iterable_identifier, + member=self.member_validator, + ) + + +def deep_iterable(member_validator, iterable_validator=None): + """ + A validator that performs deep validation of an iterable. + + :param member_validator: Validator to apply to iterable members + :param iterable_validator: Validator to apply to iterable itself + (optional) + + .. versionadded:: 19.1.0 + + :raises TypeError: if any sub-validators fail + """ + return _DeepIterable(member_validator, iterable_validator) + + +@attrs(repr=False, slots=True, hash=True) +class _DeepMapping(object): + key_validator = attrib(validator=is_callable()) + value_validator = attrib(validator=is_callable()) + mapping_validator = attrib(default=None, validator=optional(is_callable())) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.mapping_validator is not None: + self.mapping_validator(inst, attr, value) + + for key in value: + self.key_validator(inst, attr, key) + self.value_validator(inst, attr, value[key]) + + def __repr__(self): + return ( + "" + ).format(key=self.key_validator, value=self.value_validator) + + +def deep_mapping(key_validator, value_validator, mapping_validator=None): + """ + A validator that performs deep validation of a dictionary. + + :param key_validator: Validator to apply to dictionary keys + :param value_validator: Validator to apply to dictionary values + :param mapping_validator: Validator to apply to top-level mapping + attribute (optional) + + .. versionadded:: 19.1.0 + + :raises TypeError: if any sub-validators fail + """ + return _DeepMapping(key_validator, value_validator, mapping_validator) diff --git a/pipenv/vendor/attr/validators.pyi b/pipenv/vendor/attr/validators.pyi index abbaedf1..01af0684 100644 --- a/pipenv/vendor/attr/validators.pyi +++ b/pipenv/vendor/attr/validators.pyi @@ -12,3 +12,13 @@ def optional( ) -> _ValidatorType[Optional[_T]]: ... def in_(options: Container[_T]) -> _ValidatorType[_T]: ... def and_(*validators: _ValidatorType[_T]) -> _ValidatorType[_T]: ... +def deep_iterable( + member_validator: _ValidatorType[_T], + iterable_validator: Optional[_ValidatorType[_T]], +) -> _ValidatorType[_T]: ... +def deep_mapping( + key_validator: _ValidatorType[_T], + value_validator: _ValidatorType[_T], + mapping_validator: Optional[_ValidatorType[_T]], +) -> _ValidatorType[_T]: ... +def is_callable() -> _ValidatorType[_T]: ... diff --git a/pipenv/vendor/cursor/LICENSE b/pipenv/vendor/cursor/LICENSE deleted file mode 100644 index 00023c80..00000000 --- a/pipenv/vendor/cursor/LICENSE +++ /dev/null @@ -1,5 +0,0 @@ -This work is licensed under the Creative Commons -Attribution-ShareAlike 2.5 International License. To view a copy of -this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or -send a letter to Creative Commons, PO Box 1866, Mountain View, -CA 94042, USA. diff --git a/pipenv/vendor/cursor/__init__.py b/pipenv/vendor/cursor/__init__.py deleted file mode 100644 index 76a4f671..00000000 --- a/pipenv/vendor/cursor/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .cursor import hide, show, HiddenCursor - -__all__ = ["hide", "show", "HiddenCursor"] - diff --git a/pipenv/vendor/cursor/cursor.py b/pipenv/vendor/cursor/cursor.py deleted file mode 100644 index e4407c02..00000000 --- a/pipenv/vendor/cursor/cursor.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- - -## Author: James Spencer: http://stackoverflow.com/users/1375885/james-spencer -## Packager: Gijs TImmers: https://github.com/GijsTimmers - -## Based on James Spencer's answer on StackOverflow: -## http://stackoverflow.com/questions/5174810/how-to-turn-off-blinking-cursor-in-command-window - -## Licence: CC-BY-SA-2.5 -## http://creativecommons.org/licenses/by-sa/2.5/ - -## This work is licensed under the Creative Commons -## Attribution-ShareAlike 2.5 International License. To view a copy of -## this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or -## send a letter to Creative Commons, PO Box 1866, Mountain View, -## CA 94042, USA. - -import sys -import os - -if os.name == 'nt': - import ctypes - - class _CursorInfo(ctypes.Structure): - _fields_ = [("size", ctypes.c_int), - ("visible", ctypes.c_byte)] - -def hide(stream=sys.stdout): - if os.name == 'nt': - ci = _CursorInfo() - handle = ctypes.windll.kernel32.GetStdHandle(-11) - ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) - ci.visible = False - ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) - elif os.name == 'posix': - stream.write("\033[?25l") - stream.flush() - -def show(stream=sys.stdout): - if os.name == 'nt': - ci = _CursorInfo() - handle = ctypes.windll.kernel32.GetStdHandle(-11) - ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) - ci.visible = True - ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) - elif os.name == 'posix': - stream.write("\033[?25h") - stream.flush() - -class HiddenCursor(object): - def __init__(self, stream=sys.stdout): - self._stream = stream - def __enter__(self): - hide(stream=self._stream) - def __exit__(self, type, value, traceback): - show(stream=self._stream) \ No newline at end of file diff --git a/pipenv/vendor/markupsafe/LICENSE b/pipenv/vendor/markupsafe/LICENSE deleted file mode 100644 index 5d269389..00000000 --- a/pipenv/vendor/markupsafe/LICENSE +++ /dev/null @@ -1,33 +0,0 @@ -Copyright (c) 2010 by Armin Ronacher and contributors. See AUTHORS -for more details. - -Some rights reserved. - -Redistribution and use in source and binary forms of the software as well -as documentation, with or without modification, are permitted provided -that the following conditions are met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND -CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT -NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -DAMAGE. diff --git a/pipenv/vendor/markupsafe/LICENSE.rst b/pipenv/vendor/markupsafe/LICENSE.rst new file mode 100644 index 00000000..9d227a0c --- /dev/null +++ b/pipenv/vendor/markupsafe/LICENSE.rst @@ -0,0 +1,28 @@ +Copyright 2010 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/markupsafe/__init__.py b/pipenv/vendor/markupsafe/__init__.py index 68dc85f6..da05ed32 100644 --- a/pipenv/vendor/markupsafe/__init__.py +++ b/pipenv/vendor/markupsafe/__init__.py @@ -1,75 +1,74 @@ # -*- coding: utf-8 -*- """ - markupsafe - ~~~~~~~~~~ +markupsafe +~~~~~~~~~~ - Implements a Markup string. +Implements an escape function and a Markup string to replace HTML +special characters with safe representations. - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ import re import string -from collections import Mapping -from markupsafe._compat import text_type, string_types, int_types, \ - unichr, iteritems, PY2 -__version__ = "1.0" +from ._compat import int_types +from ._compat import iteritems +from ._compat import Mapping +from ._compat import PY2 +from ._compat import string_types +from ._compat import text_type +from ._compat import unichr -__all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent'] +__version__ = "1.1.1" +__all__ = ["Markup", "soft_unicode", "escape", "escape_silent"] -_striptags_re = re.compile(r'(|<[^>]*>)') -_entity_re = re.compile(r'&([^& ;]+);') +_striptags_re = re.compile(r"(|<[^>]*>)") +_entity_re = re.compile(r"&([^& ;]+);") class Markup(text_type): - r"""Marks a string as being safe for inclusion in HTML/XML output without - needing to be escaped. This implements the `__html__` interface a couple - of frameworks and web applications use. :class:`Markup` is a direct - subclass of `unicode` and provides all the methods of `unicode` just that - it escapes arguments passed and always returns `Markup`. + """A string that is ready to be safely inserted into an HTML or XML + document, either because it was escaped or because it was marked + safe. - The `escape` function returns markup objects so that double escaping can't - happen. + Passing an object to the constructor converts it to text and wraps + it to mark it safe without escaping. To escape the text, use the + :meth:`escape` class method instead. - The constructor of the :class:`Markup` class can be used for three - different things: When passed an unicode object it's assumed to be safe, - when passed an object with an HTML representation (has an `__html__` - method) that representation is used, otherwise the object passed is - converted into a unicode string and then assumed to be safe: + >>> Markup('Hello, World!') + Markup('Hello, World!') + >>> Markup(42) + Markup('42') + >>> Markup.escape('Hello, World!') + Markup('Hello <em>World</em>!') - >>> Markup("Hello World!") - Markup(u'Hello World!') - >>> class Foo(object): - ... def __html__(self): - ... return 'foo' + This implements the ``__html__()`` interface that some frameworks + use. Passing an object that implements ``__html__()`` will wrap the + output of that method, marking it safe. + + >>> class Foo: + ... def __html__(self): + ... return 'foo' ... >>> Markup(Foo()) - Markup(u'foo') + Markup('foo') - If you want object passed being always treated as unsafe you can use the - :meth:`escape` classmethod to create a :class:`Markup` object: + This is a subclass of the text type (``str`` in Python 3, + ``unicode`` in Python 2). It has the same methods as that type, but + all methods escape their arguments and return a ``Markup`` instance. - >>> Markup.escape("Hello World!") - Markup(u'Hello <em>World</em>!') - - Operations on a markup string are markup aware which means that all - arguments are passed through the :func:`escape` function: - - >>> em = Markup("%s") - >>> em % "foo & bar" - Markup(u'foo & bar') - >>> strong = Markup("%(text)s") - >>> strong % {'text': 'hacker here'} - Markup(u'<blink>hacker here</blink>') - >>> Markup("Hello ") + "" - Markup(u'Hello <foo>') + >>> Markup('%s') % 'foo & bar' + Markup('foo & bar') + >>> Markup('Hello ') + '' + Markup('Hello <foo>') """ + __slots__ = () - def __new__(cls, base=u'', encoding=None, errors='strict'): - if hasattr(base, '__html__'): + def __new__(cls, base=u"", encoding=None, errors="strict"): + if hasattr(base, "__html__"): base = base.__html__() if encoding is None: return text_type.__new__(cls, base) @@ -79,12 +78,12 @@ class Markup(text_type): return self def __add__(self, other): - if isinstance(other, string_types) or hasattr(other, '__html__'): + if isinstance(other, string_types) or hasattr(other, "__html__"): return self.__class__(super(Markup, self).__add__(self.escape(other))) return NotImplemented def __radd__(self, other): - if hasattr(other, '__html__') or isinstance(other, string_types): + if hasattr(other, "__html__") or isinstance(other, string_types): return self.escape(other).__add__(self) return NotImplemented @@ -92,6 +91,7 @@ class Markup(text_type): if isinstance(num, int_types): return self.__class__(text_type.__mul__(self, num)) return NotImplemented + __rmul__ = __mul__ def __mod__(self, arg): @@ -102,115 +102,124 @@ class Markup(text_type): return self.__class__(text_type.__mod__(self, arg)) def __repr__(self): - return '%s(%s)' % ( - self.__class__.__name__, - text_type.__repr__(self) - ) + return "%s(%s)" % (self.__class__.__name__, text_type.__repr__(self)) def join(self, seq): return self.__class__(text_type.join(self, map(self.escape, seq))) + join.__doc__ = text_type.join.__doc__ def split(self, *args, **kwargs): return list(map(self.__class__, text_type.split(self, *args, **kwargs))) + split.__doc__ = text_type.split.__doc__ def rsplit(self, *args, **kwargs): return list(map(self.__class__, text_type.rsplit(self, *args, **kwargs))) + rsplit.__doc__ = text_type.rsplit.__doc__ def splitlines(self, *args, **kwargs): - return list(map(self.__class__, text_type.splitlines( - self, *args, **kwargs))) + return list(map(self.__class__, text_type.splitlines(self, *args, **kwargs))) + splitlines.__doc__ = text_type.splitlines.__doc__ def unescape(self): - r"""Unescape markup again into an text_type string. This also resolves - known HTML4 and XHTML entities: + """Convert escaped markup back into a text string. This replaces + HTML entities with the characters they represent. - >>> Markup("Main » About").unescape() - u'Main \xbb About' + >>> Markup('Main » About').unescape() + 'Main » About' """ - from markupsafe._constants import HTML_ENTITIES + from ._constants import HTML_ENTITIES + def handle_match(m): name = m.group(1) if name in HTML_ENTITIES: return unichr(HTML_ENTITIES[name]) try: - if name[:2] in ('#x', '#X'): + if name[:2] in ("#x", "#X"): return unichr(int(name[2:], 16)) - elif name.startswith('#'): + elif name.startswith("#"): return unichr(int(name[1:])) except ValueError: pass # Don't modify unexpected input. return m.group() + return _entity_re.sub(handle_match, text_type(self)) def striptags(self): - r"""Unescape markup into an text_type string and strip all tags. This - also resolves known HTML4 and XHTML entities. Whitespace is - normalized to one: + """:meth:`unescape` the markup, remove tags, and normalize + whitespace to single spaces. - >>> Markup("Main » About").striptags() - u'Main \xbb About' + >>> Markup('Main »\tAbout').striptags() + 'Main » About' """ - stripped = u' '.join(_striptags_re.sub('', self).split()) + stripped = u" ".join(_striptags_re.sub("", self).split()) return Markup(stripped).unescape() @classmethod def escape(cls, s): - """Escape the string. Works like :func:`escape` with the difference - that for subclasses of :class:`Markup` this function would return the - correct subclass. + """Escape a string. Calls :func:`escape` and ensures that for + subclasses the correct type is returned. """ rv = escape(s) if rv.__class__ is not cls: return cls(rv) return rv - def make_simple_escaping_wrapper(name): + def make_simple_escaping_wrapper(name): # noqa: B902 orig = getattr(text_type, name) + def func(self, *args, **kwargs): args = _escape_argspec(list(args), enumerate(args), self.escape) _escape_argspec(kwargs, iteritems(kwargs), self.escape) return self.__class__(orig(self, *args, **kwargs)) + func.__name__ = orig.__name__ func.__doc__ = orig.__doc__ return func - for method in '__getitem__', 'capitalize', \ - 'title', 'lower', 'upper', 'replace', 'ljust', \ - 'rjust', 'lstrip', 'rstrip', 'center', 'strip', \ - 'translate', 'expandtabs', 'swapcase', 'zfill': + for method in ( + "__getitem__", + "capitalize", + "title", + "lower", + "upper", + "replace", + "ljust", + "rjust", + "lstrip", + "rstrip", + "center", + "strip", + "translate", + "expandtabs", + "swapcase", + "zfill", + ): locals()[method] = make_simple_escaping_wrapper(method) - # new in python 2.5 - if hasattr(text_type, 'partition'): - def partition(self, sep): - return tuple(map(self.__class__, - text_type.partition(self, self.escape(sep)))) - def rpartition(self, sep): - return tuple(map(self.__class__, - text_type.rpartition(self, self.escape(sep)))) + def partition(self, sep): + return tuple(map(self.__class__, text_type.partition(self, self.escape(sep)))) - # new in python 2.6 - if hasattr(text_type, 'format'): - def format(*args, **kwargs): - self, args = args[0], args[1:] - formatter = EscapeFormatter(self.escape) - kwargs = _MagicFormatMapping(args, kwargs) - return self.__class__(formatter.vformat(self, args, kwargs)) + def rpartition(self, sep): + return tuple(map(self.__class__, text_type.rpartition(self, self.escape(sep)))) - def __html_format__(self, format_spec): - if format_spec: - raise ValueError('Unsupported format specification ' - 'for Markup.') - return self + def format(self, *args, **kwargs): + formatter = EscapeFormatter(self.escape) + kwargs = _MagicFormatMapping(args, kwargs) + return self.__class__(formatter.vformat(self, args, kwargs)) + + def __html_format__(self, format_spec): + if format_spec: + raise ValueError("Unsupported format specification " "for Markup.") + return self # not in python 3 - if hasattr(text_type, '__getslice__'): - __getslice__ = make_simple_escaping_wrapper('__getslice__') + if hasattr(text_type, "__getslice__"): + __getslice__ = make_simple_escaping_wrapper("__getslice__") del method, make_simple_escaping_wrapper @@ -229,7 +238,7 @@ class _MagicFormatMapping(Mapping): self._last_index = 0 def __getitem__(self, key): - if key == '': + if key == "": idx = self._last_index self._last_index += 1 try: @@ -246,35 +255,37 @@ class _MagicFormatMapping(Mapping): return len(self._kwargs) -if hasattr(text_type, 'format'): - class EscapeFormatter(string.Formatter): +if hasattr(text_type, "format"): + class EscapeFormatter(string.Formatter): def __init__(self, escape): self.escape = escape def format_field(self, value, format_spec): - if hasattr(value, '__html_format__'): + if hasattr(value, "__html_format__"): rv = value.__html_format__(format_spec) - elif hasattr(value, '__html__'): + elif hasattr(value, "__html__"): if format_spec: - raise ValueError('No format specification allowed ' - 'when formatting an object with ' - 'its __html__ method.') + raise ValueError( + "Format specifier {0} given, but {1} does not" + " define __html_format__. A class that defines" + " __html__ must define __html_format__ to work" + " with format specifiers.".format(format_spec, type(value)) + ) rv = value.__html__() else: # We need to make sure the format spec is unicode here as # otherwise the wrong callback methods are invoked. For # instance a byte string there would invoke __str__ and # not __unicode__. - rv = string.Formatter.format_field( - self, value, text_type(format_spec)) + rv = string.Formatter.format_field(self, value, text_type(format_spec)) return text_type(self.escape(rv)) def _escape_argspec(obj, iterable, escape): """Helper for various string-wrapped functions.""" for key, value in iterable: - if hasattr(value, '__html__') or isinstance(value, string_types): + if hasattr(value, "__html__") or isinstance(value, string_types): obj[key] = escape(value) return obj @@ -286,20 +297,31 @@ class _MarkupEscapeHelper(object): self.obj = obj self.escape = escape - __getitem__ = lambda s, x: _MarkupEscapeHelper(s.obj[x], s.escape) - __unicode__ = __str__ = lambda s: text_type(s.escape(s.obj)) - __repr__ = lambda s: str(s.escape(repr(s.obj))) - __int__ = lambda s: int(s.obj) - __float__ = lambda s: float(s.obj) + def __getitem__(self, item): + return _MarkupEscapeHelper(self.obj[item], self.escape) + + def __str__(self): + return text_type(self.escape(self.obj)) + + __unicode__ = __str__ + + def __repr__(self): + return str(self.escape(repr(self.obj))) + + def __int__(self): + return int(self.obj) + + def __float__(self): + return float(self.obj) # we have to import it down here as the speedups and native # modules imports the markup type which is define above. try: - from markupsafe._speedups import escape, escape_silent, soft_unicode + from ._speedups import escape, escape_silent, soft_unicode except ImportError: - from markupsafe._native import escape, escape_silent, soft_unicode + from ._native import escape, escape_silent, soft_unicode if not PY2: soft_str = soft_unicode - __all__.append('soft_str') + __all__.append("soft_str") diff --git a/pipenv/vendor/markupsafe/_compat.py b/pipenv/vendor/markupsafe/_compat.py index 62e5632a..bc05090f 100644 --- a/pipenv/vendor/markupsafe/_compat.py +++ b/pipenv/vendor/markupsafe/_compat.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- """ - markupsafe._compat - ~~~~~~~~~~~~~~~~~~ +markupsafe._compat +~~~~~~~~~~~~~~~~~~ - Compatibility module for different Python versions. - - :copyright: (c) 2013 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ import sys @@ -17,10 +15,19 @@ if not PY2: string_types = (str,) unichr = chr int_types = (int,) - iteritems = lambda x: iter(x.items()) + + def iteritems(x): + return iter(x.items()) + + from collections.abc import Mapping + else: text_type = unicode string_types = (str, unicode) unichr = unichr int_types = (int, long) - iteritems = lambda x: x.iteritems() + + def iteritems(x): + return x.iteritems() + + from collections import Mapping diff --git a/pipenv/vendor/markupsafe/_constants.py b/pipenv/vendor/markupsafe/_constants.py index 919bf03c..7c57c2d2 100644 --- a/pipenv/vendor/markupsafe/_constants.py +++ b/pipenv/vendor/markupsafe/_constants.py @@ -1,267 +1,264 @@ # -*- coding: utf-8 -*- """ - markupsafe._constants - ~~~~~~~~~~~~~~~~~~~~~ +markupsafe._constants +~~~~~~~~~~~~~~~~~~~~~ - Highlevel implementation of the Markup string. - - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ - HTML_ENTITIES = { - 'AElig': 198, - 'Aacute': 193, - 'Acirc': 194, - 'Agrave': 192, - 'Alpha': 913, - 'Aring': 197, - 'Atilde': 195, - 'Auml': 196, - 'Beta': 914, - 'Ccedil': 199, - 'Chi': 935, - 'Dagger': 8225, - 'Delta': 916, - 'ETH': 208, - 'Eacute': 201, - 'Ecirc': 202, - 'Egrave': 200, - 'Epsilon': 917, - 'Eta': 919, - 'Euml': 203, - 'Gamma': 915, - 'Iacute': 205, - 'Icirc': 206, - 'Igrave': 204, - 'Iota': 921, - 'Iuml': 207, - 'Kappa': 922, - 'Lambda': 923, - 'Mu': 924, - 'Ntilde': 209, - 'Nu': 925, - 'OElig': 338, - 'Oacute': 211, - 'Ocirc': 212, - 'Ograve': 210, - 'Omega': 937, - 'Omicron': 927, - 'Oslash': 216, - 'Otilde': 213, - 'Ouml': 214, - 'Phi': 934, - 'Pi': 928, - 'Prime': 8243, - 'Psi': 936, - 'Rho': 929, - 'Scaron': 352, - 'Sigma': 931, - 'THORN': 222, - 'Tau': 932, - 'Theta': 920, - 'Uacute': 218, - 'Ucirc': 219, - 'Ugrave': 217, - 'Upsilon': 933, - 'Uuml': 220, - 'Xi': 926, - 'Yacute': 221, - 'Yuml': 376, - 'Zeta': 918, - 'aacute': 225, - 'acirc': 226, - 'acute': 180, - 'aelig': 230, - 'agrave': 224, - 'alefsym': 8501, - 'alpha': 945, - 'amp': 38, - 'and': 8743, - 'ang': 8736, - 'apos': 39, - 'aring': 229, - 'asymp': 8776, - 'atilde': 227, - 'auml': 228, - 'bdquo': 8222, - 'beta': 946, - 'brvbar': 166, - 'bull': 8226, - 'cap': 8745, - 'ccedil': 231, - 'cedil': 184, - 'cent': 162, - 'chi': 967, - 'circ': 710, - 'clubs': 9827, - 'cong': 8773, - 'copy': 169, - 'crarr': 8629, - 'cup': 8746, - 'curren': 164, - 'dArr': 8659, - 'dagger': 8224, - 'darr': 8595, - 'deg': 176, - 'delta': 948, - 'diams': 9830, - 'divide': 247, - 'eacute': 233, - 'ecirc': 234, - 'egrave': 232, - 'empty': 8709, - 'emsp': 8195, - 'ensp': 8194, - 'epsilon': 949, - 'equiv': 8801, - 'eta': 951, - 'eth': 240, - 'euml': 235, - 'euro': 8364, - 'exist': 8707, - 'fnof': 402, - 'forall': 8704, - 'frac12': 189, - 'frac14': 188, - 'frac34': 190, - 'frasl': 8260, - 'gamma': 947, - 'ge': 8805, - 'gt': 62, - 'hArr': 8660, - 'harr': 8596, - 'hearts': 9829, - 'hellip': 8230, - 'iacute': 237, - 'icirc': 238, - 'iexcl': 161, - 'igrave': 236, - 'image': 8465, - 'infin': 8734, - 'int': 8747, - 'iota': 953, - 'iquest': 191, - 'isin': 8712, - 'iuml': 239, - 'kappa': 954, - 'lArr': 8656, - 'lambda': 955, - 'lang': 9001, - 'laquo': 171, - 'larr': 8592, - 'lceil': 8968, - 'ldquo': 8220, - 'le': 8804, - 'lfloor': 8970, - 'lowast': 8727, - 'loz': 9674, - 'lrm': 8206, - 'lsaquo': 8249, - 'lsquo': 8216, - 'lt': 60, - 'macr': 175, - 'mdash': 8212, - 'micro': 181, - 'middot': 183, - 'minus': 8722, - 'mu': 956, - 'nabla': 8711, - 'nbsp': 160, - 'ndash': 8211, - 'ne': 8800, - 'ni': 8715, - 'not': 172, - 'notin': 8713, - 'nsub': 8836, - 'ntilde': 241, - 'nu': 957, - 'oacute': 243, - 'ocirc': 244, - 'oelig': 339, - 'ograve': 242, - 'oline': 8254, - 'omega': 969, - 'omicron': 959, - 'oplus': 8853, - 'or': 8744, - 'ordf': 170, - 'ordm': 186, - 'oslash': 248, - 'otilde': 245, - 'otimes': 8855, - 'ouml': 246, - 'para': 182, - 'part': 8706, - 'permil': 8240, - 'perp': 8869, - 'phi': 966, - 'pi': 960, - 'piv': 982, - 'plusmn': 177, - 'pound': 163, - 'prime': 8242, - 'prod': 8719, - 'prop': 8733, - 'psi': 968, - 'quot': 34, - 'rArr': 8658, - 'radic': 8730, - 'rang': 9002, - 'raquo': 187, - 'rarr': 8594, - 'rceil': 8969, - 'rdquo': 8221, - 'real': 8476, - 'reg': 174, - 'rfloor': 8971, - 'rho': 961, - 'rlm': 8207, - 'rsaquo': 8250, - 'rsquo': 8217, - 'sbquo': 8218, - 'scaron': 353, - 'sdot': 8901, - 'sect': 167, - 'shy': 173, - 'sigma': 963, - 'sigmaf': 962, - 'sim': 8764, - 'spades': 9824, - 'sub': 8834, - 'sube': 8838, - 'sum': 8721, - 'sup': 8835, - 'sup1': 185, - 'sup2': 178, - 'sup3': 179, - 'supe': 8839, - 'szlig': 223, - 'tau': 964, - 'there4': 8756, - 'theta': 952, - 'thetasym': 977, - 'thinsp': 8201, - 'thorn': 254, - 'tilde': 732, - 'times': 215, - 'trade': 8482, - 'uArr': 8657, - 'uacute': 250, - 'uarr': 8593, - 'ucirc': 251, - 'ugrave': 249, - 'uml': 168, - 'upsih': 978, - 'upsilon': 965, - 'uuml': 252, - 'weierp': 8472, - 'xi': 958, - 'yacute': 253, - 'yen': 165, - 'yuml': 255, - 'zeta': 950, - 'zwj': 8205, - 'zwnj': 8204 + "AElig": 198, + "Aacute": 193, + "Acirc": 194, + "Agrave": 192, + "Alpha": 913, + "Aring": 197, + "Atilde": 195, + "Auml": 196, + "Beta": 914, + "Ccedil": 199, + "Chi": 935, + "Dagger": 8225, + "Delta": 916, + "ETH": 208, + "Eacute": 201, + "Ecirc": 202, + "Egrave": 200, + "Epsilon": 917, + "Eta": 919, + "Euml": 203, + "Gamma": 915, + "Iacute": 205, + "Icirc": 206, + "Igrave": 204, + "Iota": 921, + "Iuml": 207, + "Kappa": 922, + "Lambda": 923, + "Mu": 924, + "Ntilde": 209, + "Nu": 925, + "OElig": 338, + "Oacute": 211, + "Ocirc": 212, + "Ograve": 210, + "Omega": 937, + "Omicron": 927, + "Oslash": 216, + "Otilde": 213, + "Ouml": 214, + "Phi": 934, + "Pi": 928, + "Prime": 8243, + "Psi": 936, + "Rho": 929, + "Scaron": 352, + "Sigma": 931, + "THORN": 222, + "Tau": 932, + "Theta": 920, + "Uacute": 218, + "Ucirc": 219, + "Ugrave": 217, + "Upsilon": 933, + "Uuml": 220, + "Xi": 926, + "Yacute": 221, + "Yuml": 376, + "Zeta": 918, + "aacute": 225, + "acirc": 226, + "acute": 180, + "aelig": 230, + "agrave": 224, + "alefsym": 8501, + "alpha": 945, + "amp": 38, + "and": 8743, + "ang": 8736, + "apos": 39, + "aring": 229, + "asymp": 8776, + "atilde": 227, + "auml": 228, + "bdquo": 8222, + "beta": 946, + "brvbar": 166, + "bull": 8226, + "cap": 8745, + "ccedil": 231, + "cedil": 184, + "cent": 162, + "chi": 967, + "circ": 710, + "clubs": 9827, + "cong": 8773, + "copy": 169, + "crarr": 8629, + "cup": 8746, + "curren": 164, + "dArr": 8659, + "dagger": 8224, + "darr": 8595, + "deg": 176, + "delta": 948, + "diams": 9830, + "divide": 247, + "eacute": 233, + "ecirc": 234, + "egrave": 232, + "empty": 8709, + "emsp": 8195, + "ensp": 8194, + "epsilon": 949, + "equiv": 8801, + "eta": 951, + "eth": 240, + "euml": 235, + "euro": 8364, + "exist": 8707, + "fnof": 402, + "forall": 8704, + "frac12": 189, + "frac14": 188, + "frac34": 190, + "frasl": 8260, + "gamma": 947, + "ge": 8805, + "gt": 62, + "hArr": 8660, + "harr": 8596, + "hearts": 9829, + "hellip": 8230, + "iacute": 237, + "icirc": 238, + "iexcl": 161, + "igrave": 236, + "image": 8465, + "infin": 8734, + "int": 8747, + "iota": 953, + "iquest": 191, + "isin": 8712, + "iuml": 239, + "kappa": 954, + "lArr": 8656, + "lambda": 955, + "lang": 9001, + "laquo": 171, + "larr": 8592, + "lceil": 8968, + "ldquo": 8220, + "le": 8804, + "lfloor": 8970, + "lowast": 8727, + "loz": 9674, + "lrm": 8206, + "lsaquo": 8249, + "lsquo": 8216, + "lt": 60, + "macr": 175, + "mdash": 8212, + "micro": 181, + "middot": 183, + "minus": 8722, + "mu": 956, + "nabla": 8711, + "nbsp": 160, + "ndash": 8211, + "ne": 8800, + "ni": 8715, + "not": 172, + "notin": 8713, + "nsub": 8836, + "ntilde": 241, + "nu": 957, + "oacute": 243, + "ocirc": 244, + "oelig": 339, + "ograve": 242, + "oline": 8254, + "omega": 969, + "omicron": 959, + "oplus": 8853, + "or": 8744, + "ordf": 170, + "ordm": 186, + "oslash": 248, + "otilde": 245, + "otimes": 8855, + "ouml": 246, + "para": 182, + "part": 8706, + "permil": 8240, + "perp": 8869, + "phi": 966, + "pi": 960, + "piv": 982, + "plusmn": 177, + "pound": 163, + "prime": 8242, + "prod": 8719, + "prop": 8733, + "psi": 968, + "quot": 34, + "rArr": 8658, + "radic": 8730, + "rang": 9002, + "raquo": 187, + "rarr": 8594, + "rceil": 8969, + "rdquo": 8221, + "real": 8476, + "reg": 174, + "rfloor": 8971, + "rho": 961, + "rlm": 8207, + "rsaquo": 8250, + "rsquo": 8217, + "sbquo": 8218, + "scaron": 353, + "sdot": 8901, + "sect": 167, + "shy": 173, + "sigma": 963, + "sigmaf": 962, + "sim": 8764, + "spades": 9824, + "sub": 8834, + "sube": 8838, + "sum": 8721, + "sup": 8835, + "sup1": 185, + "sup2": 178, + "sup3": 179, + "supe": 8839, + "szlig": 223, + "tau": 964, + "there4": 8756, + "theta": 952, + "thetasym": 977, + "thinsp": 8201, + "thorn": 254, + "tilde": 732, + "times": 215, + "trade": 8482, + "uArr": 8657, + "uacute": 250, + "uarr": 8593, + "ucirc": 251, + "ugrave": 249, + "uml": 168, + "upsih": 978, + "upsilon": 965, + "uuml": 252, + "weierp": 8472, + "xi": 958, + "yacute": 253, + "yen": 165, + "yuml": 255, + "zeta": 950, + "zwj": 8205, + "zwnj": 8204, } diff --git a/pipenv/vendor/markupsafe/_native.py b/pipenv/vendor/markupsafe/_native.py index 5e83f10a..cd08752c 100644 --- a/pipenv/vendor/markupsafe/_native.py +++ b/pipenv/vendor/markupsafe/_native.py @@ -1,36 +1,49 @@ # -*- coding: utf-8 -*- """ - markupsafe._native - ~~~~~~~~~~~~~~~~~~ +markupsafe._native +~~~~~~~~~~~~~~~~~~ - Native Python implementation the C module is not compiled. +Native Python implementation used when the C module is not compiled. - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ -from markupsafe import Markup -from markupsafe._compat import text_type +from . import Markup +from ._compat import text_type def escape(s): - """Convert the characters &, <, >, ' and " in string s to HTML-safe - sequences. Use this if you need to display text that might contain - such characters in HTML. Marks return value as markup string. + """Replace the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in + the string with HTML-safe sequences. Use this if you need to display + text that might contain such characters in HTML. + + If the object has an ``__html__`` method, it is called and the + return value is assumed to already be safe for HTML. + + :param s: An object to be converted to a string and escaped. + :return: A :class:`Markup` string with the escaped text. """ - if hasattr(s, '__html__'): - return s.__html__() - return Markup(text_type(s) - .replace('&', '&') - .replace('>', '>') - .replace('<', '<') - .replace("'", ''') - .replace('"', '"') + if hasattr(s, "__html__"): + return Markup(s.__html__()) + return Markup( + text_type(s) + .replace("&", "&") + .replace(">", ">") + .replace("<", "<") + .replace("'", "'") + .replace('"', """) ) def escape_silent(s): - """Like :func:`escape` but converts `None` into an empty - markup string. + """Like :func:`escape` but treats ``None`` as the empty string. + Useful with optional values, as otherwise you get the string + ``'None'`` when the value is ``None``. + + >>> escape(None) + Markup('None') + >>> escape_silent(None) + Markup('') """ if s is None: return Markup() @@ -38,8 +51,18 @@ def escape_silent(s): def soft_unicode(s): - """Make a string unicode if it isn't already. That way a markup - string is not converted back to unicode. + """Convert an object to a string if it isn't already. This preserves + a :class:`Markup` string rather than converting it back to a basic + string, so it will still be marked as safe and won't be escaped + again. + + >>> value = escape('') + >>> value + Markup('<User 1>') + >>> escape(str(value)) + Markup('&lt;User 1&gt;') + >>> escape(soft_unicode(value)) + Markup('<User 1>') """ if not isinstance(s, text_type): s = text_type(s) diff --git a/pipenv/vendor/markupsafe/_speedups.c b/pipenv/vendor/markupsafe/_speedups.c index d779a68c..12d2c4a7 100644 --- a/pipenv/vendor/markupsafe/_speedups.c +++ b/pipenv/vendor/markupsafe/_speedups.c @@ -2,33 +2,30 @@ * markupsafe._speedups * ~~~~~~~~~~~~~~~~~~~~ * - * This module implements functions for automatic escaping in C for better - * performance. + * C implementation of escaping for better performance. Used instead of + * the native Python implementation when compiled. * - * :copyright: (c) 2010 by Armin Ronacher. - * :license: BSD. + * :copyright: 2010 Pallets + * :license: BSD-3-Clause */ - #include +#if PY_MAJOR_VERSION < 3 #define ESCAPED_CHARS_TABLE_SIZE 63 #define UNICHR(x) (PyUnicode_AS_UNICODE((PyUnicodeObject*)PyUnicode_DecodeASCII(x, strlen(x), NULL))); -#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) -typedef int Py_ssize_t; -#define PY_SSIZE_T_MAX INT_MAX -#define PY_SSIZE_T_MIN INT_MIN -#endif - - -static PyObject* markup; static Py_ssize_t escaped_chars_delta_len[ESCAPED_CHARS_TABLE_SIZE]; static Py_UNICODE *escaped_chars_repl[ESCAPED_CHARS_TABLE_SIZE]; +#endif + +static PyObject* markup; static int init_constants(void) { PyObject *module; + +#if PY_MAJOR_VERSION < 3 /* mapping of characters to replace */ escaped_chars_repl['"'] = UNICHR("""); escaped_chars_repl['\''] = UNICHR("'"); @@ -41,6 +38,7 @@ init_constants(void) escaped_chars_delta_len['"'] = escaped_chars_delta_len['\''] = \ escaped_chars_delta_len['&'] = 4; escaped_chars_delta_len['<'] = escaped_chars_delta_len['>'] = 3; +#endif /* import markup type so that we can mark the return value */ module = PyImport_ImportModule("markupsafe"); @@ -52,6 +50,7 @@ init_constants(void) return 1; } +#if PY_MAJOR_VERSION < 3 static PyObject* escape_unicode(PyUnicodeObject *in) { @@ -112,13 +111,192 @@ escape_unicode(PyUnicodeObject *in) return (PyObject*)out; } +#else /* PY_MAJOR_VERSION < 3 */ +#define GET_DELTA(inp, inp_end, delta) \ + while (inp < inp_end) { \ + switch (*inp++) { \ + case '"': \ + case '\'': \ + case '&': \ + delta += 4; \ + break; \ + case '<': \ + case '>': \ + delta += 3; \ + break; \ + } \ + } + +#define DO_ESCAPE(inp, inp_end, outp) \ + { \ + Py_ssize_t ncopy = 0; \ + while (inp < inp_end) { \ + switch (*inp) { \ + case '"': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = '#'; \ + *outp++ = '3'; \ + *outp++ = '4'; \ + *outp++ = ';'; \ + break; \ + case '\'': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = '#'; \ + *outp++ = '3'; \ + *outp++ = '9'; \ + *outp++ = ';'; \ + break; \ + case '&': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'a'; \ + *outp++ = 'm'; \ + *outp++ = 'p'; \ + *outp++ = ';'; \ + break; \ + case '<': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'l'; \ + *outp++ = 't'; \ + *outp++ = ';'; \ + break; \ + case '>': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'g'; \ + *outp++ = 't'; \ + *outp++ = ';'; \ + break; \ + default: \ + ncopy++; \ + } \ + inp++; \ + } \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + } + +static PyObject* +escape_unicode_kind1(PyUnicodeObject *in) +{ + Py_UCS1 *inp = PyUnicode_1BYTE_DATA(in); + Py_UCS1 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS1 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, + PyUnicode_IS_ASCII(in) ? 127 : 255); + if (!out) + return NULL; + + inp = PyUnicode_1BYTE_DATA(in); + outp = PyUnicode_1BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + +static PyObject* +escape_unicode_kind2(PyUnicodeObject *in) +{ + Py_UCS2 *inp = PyUnicode_2BYTE_DATA(in); + Py_UCS2 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS2 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, 65535); + if (!out) + return NULL; + + inp = PyUnicode_2BYTE_DATA(in); + outp = PyUnicode_2BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + + +static PyObject* +escape_unicode_kind4(PyUnicodeObject *in) +{ + Py_UCS4 *inp = PyUnicode_4BYTE_DATA(in); + Py_UCS4 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS4 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, 1114111); + if (!out) + return NULL; + + inp = PyUnicode_4BYTE_DATA(in); + outp = PyUnicode_4BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + +static PyObject* +escape_unicode(PyUnicodeObject *in) +{ + if (PyUnicode_READY(in)) + return NULL; + + switch (PyUnicode_KIND(in)) { + case PyUnicode_1BYTE_KIND: + return escape_unicode_kind1(in); + case PyUnicode_2BYTE_KIND: + return escape_unicode_kind2(in); + case PyUnicode_4BYTE_KIND: + return escape_unicode_kind4(in); + } + assert(0); /* shouldn't happen */ + return NULL; +} +#endif /* PY_MAJOR_VERSION < 3 */ static PyObject* escape(PyObject *self, PyObject *text) { + static PyObject *id_html; PyObject *s = NULL, *rv = NULL, *html; + if (id_html == NULL) { +#if PY_MAJOR_VERSION < 3 + id_html = PyString_InternFromString("__html__"); +#else + id_html = PyUnicode_InternFromString("__html__"); +#endif + if (id_html == NULL) { + return NULL; + } + } + /* we don't have to escape integers, bools or floats */ if (PyLong_CheckExact(text) || #if PY_MAJOR_VERSION < 3 @@ -129,10 +307,16 @@ escape(PyObject *self, PyObject *text) return PyObject_CallFunctionObjArgs(markup, text, NULL); /* if the object has an __html__ method that performs the escaping */ - html = PyObject_GetAttrString(text, "__html__"); + html = PyObject_GetAttr(text ,id_html); if (html) { - rv = PyObject_CallObject(html, NULL); + s = PyObject_CallObject(html, NULL); Py_DECREF(html); + if (s == NULL) { + return NULL; + } + /* Convert to Markup object */ + rv = PyObject_CallFunctionObjArgs(markup, (PyObject*)s, NULL); + Py_DECREF(s); return rv; } diff --git a/pipenv/vendor/orderedmultidict/LICENSE b/pipenv/vendor/orderedmultidict/LICENSE.md old mode 100755 new mode 100644 similarity index 95% rename from pipenv/vendor/orderedmultidict/LICENSE rename to pipenv/vendor/orderedmultidict/LICENSE.md index fd832f40..210e8658 --- a/pipenv/vendor/orderedmultidict/LICENSE +++ b/pipenv/vendor/orderedmultidict/LICENSE.md @@ -28,4 +28,4 @@ OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\ SOFTWARE. -For more information, please refer to +For more information, please refer to diff --git a/pipenv/vendor/parse.py b/pipenv/vendor/parse.py index 7f9f0786..b5d543f9 100644 --- a/pipenv/vendor/parse.py +++ b/pipenv/vendor/parse.py @@ -3,7 +3,7 @@ r'''Parse strings using a specification based on the Python format() syntax. ``parse()`` is the opposite of ``format()`` The module is set up to only export ``parse()``, ``search()``, ``findall()``, -and ``with_pattern()`` when ``import *`` is used: +and ``with_pattern()`` when ``import \*`` is used: >>> from parse import * @@ -78,9 +78,11 @@ Some simple parse() format string examples: {'item': 'hand grenade'} >>> print(r['item']) hand grenade +>>> 'item' in r +True -Dotted names and indexes are possible though the application must make -additional sense of the result: +Note that `in` only works if you have named fields. Dotted names and indexes +are possible though the application must make additional sense of the result: >>> r = parse("Mmm, {food.type}, I love it!", "Mmm, spam, I love it!") >>> print(r) @@ -132,38 +134,39 @@ The differences between `parse()` and `format()` are: ===== =========================================== ======== Type Characters Matched Output ===== =========================================== ======== - w Letters and underscore str - W Non-letter and underscore str - s Whitespace str - S Non-whitespace str - d Digits (effectively integer numbers) int - D Non-digit str - n Numbers with thousands separators (, or .) int - % Percentage (converted to value/100.0) float - f Fixed-point numbers float - F Decimal numbers Decimal - e Floating-point numbers with exponent float +l Letters (ASCII) str +w Letters, numbers and underscore str +W Not letters, numbers and underscore str +s Whitespace str +S Non-whitespace str +d Digits (effectively integer numbers) int +D Non-digit str +n Numbers with thousands separators (, or .) int +% Percentage (converted to value/100.0) float +f Fixed-point numbers float +F Decimal numbers Decimal +e Floating-point numbers with exponent float e.g. 1.1e-10, NAN (all case insensitive) - g General number format (either d, f or e) float - b Binary numbers int - o Octal numbers int - x Hexadecimal numbers (lower and upper case) int - ti ISO 8601 format date/time datetime +g General number format (either d, f or e) float +b Binary numbers int +o Octal numbers int +x Hexadecimal numbers (lower and upper case) int +ti ISO 8601 format date/time datetime e.g. 1972-01-20T10:21:36Z ("T" and "Z" optional) - te RFC2822 e-mail format date/time datetime +te RFC2822 e-mail format date/time datetime e.g. Mon, 20 Jan 1972 10:21:36 +1000 - tg Global (day/month) format date/time datetime +tg Global (day/month) format date/time datetime e.g. 20/1/1972 10:21:36 AM +1:00 - ta US (month/day) format date/time datetime +ta US (month/day) format date/time datetime e.g. 1/20/1972 10:21:36 PM +10:30 - tc ctime() format date/time datetime +tc ctime() format date/time datetime e.g. Sun Sep 16 01:03:52 1973 - th HTTP log format date/time datetime +th HTTP log format date/time datetime e.g. 21/Nov/2011:00:07:11 +0000 - ts Linux system log format date/time datetime +ts Linux system log format date/time datetime e.g. Nov 9 03:37:44 - tt Time time +tt Time time e.g. 10:21:36 PM -5:30 ===== =========================================== ======== @@ -342,6 +345,13 @@ the pattern, the actual match represents the shortest successful match for **Version history (in brief)**: +- 1.11.1 Revert having unicode char in docstring, it breaks Bamboo builds(?!) +- 1.11.0 Implement `__contains__` for Result instances. +- 1.10.0 Introduce a "letters" matcher, since "w" matches numbers + also. +- 1.9.1 Fix deprecation warnings around backslashes in regex strings + (thanks Mickael Schoentgen). Also fix some documentation formatting + issues. - 1.9.0 We now honor precision and width specifiers when parsing numbers and strings, allowing parsing of concatenated elements of fixed width (thanks Julia Signell) @@ -400,12 +410,12 @@ the pattern, the actual match represents the shortest successful match for and removed the restriction on mixing fixed-position and named fields - 1.0.0 initial release -This code is copyright 2012-2017 Richard Jones +This code is copyright 2012-2019 Richard Jones See the end of the source file for the license of use. ''' from __future__ import absolute_import -__version__ = '1.9.0' +__version__ = '1.11.1' # yes, I now have two problems import re @@ -530,9 +540,9 @@ MONTHS_MAP = dict( Nov=11, November=11, Dec=12, December=12 ) -DAYS_PAT = '(Mon|Tue|Wed|Thu|Fri|Sat|Sun)' -MONTHS_PAT = '(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)' -ALL_MONTHS_PAT = '(%s)' % '|'.join(MONTHS_MAP) +DAYS_PAT = r'(Mon|Tue|Wed|Thu|Fri|Sat|Sun)' +MONTHS_PAT = r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)' +ALL_MONTHS_PAT = r'(%s)' % '|'.join(MONTHS_MAP) TIME_PAT = r'(\d{1,2}:\d{1,2}(:\d{1,2}(\.\d+)?)?)' AM_PAT = r'(\s+[AP]M)' TZ_PAT = r'(\s+[-+]\d\d?:?\d\d)' @@ -550,11 +560,11 @@ def date_convert(string, match, ymd=None, mdy=None, dmy=None, m=groups[mm] d=groups[dd] elif ymd is not None: - y, m, d = re.split('[-/\s]', groups[ymd]) + y, m, d = re.split(r'[-/\s]', groups[ymd]) elif mdy is not None: - m, d, y = re.split('[-/\s]', groups[mdy]) + m, d, y = re.split(r'[-/\s]', groups[mdy]) elif dmy is not None: - d, m, y = re.split('[-/\s]', groups[dmy]) + d, m, y = re.split(r'[-/\s]', groups[dmy]) elif d_m_y is not None: d, m, y = d_m_y d = groups[d] @@ -636,10 +646,10 @@ class RepeatedNameError(ValueError): # note: {} are handled separately # note: I don't use r'' here because Sublime Text 2 syntax highlight has a fit -REGEX_SAFETY = re.compile('([?\\\\.[\]()*+\^$!\|])') +REGEX_SAFETY = re.compile(r'([?\\\\.[\]()*+\^$!\|])') # allowed field types -ALLOWED_TYPES = set(list('nbox%fFegwWdDsS') + +ALLOWED_TYPES = set(list('nbox%fFegwWdDsSl') + ['t' + c for c in 'ieahgcts']) @@ -745,7 +755,7 @@ class Parser(object): @property def _match_re(self): if self.__match_re is None: - expression = '^%s$' % self._expression + expression = r'^%s$' % self._expression try: self.__match_re = re.compile(expression, self._re_flags) except AssertionError: @@ -923,16 +933,16 @@ class Parser(object): name, self._name_types[name])) group = self._name_to_group_map[name] # match previously-seen value - return '(?P=%s)' % group + return r'(?P=%s)' % group else: group = self._to_group_name(name) self._name_types[name] = format self._named_fields.append(group) # this will become a group, which must not contain dots - wrap = '(?P<%s>%%s)' % group + wrap = r'(?P<%s>%%s)' % group else: self._fixed_fields.append(self._group_index) - wrap = '(%s)' + wrap = r'(%s)' if ':' in field: format = field[1:] group = self._group_index @@ -940,7 +950,7 @@ class Parser(object): # simplest case: no type specifier ({} or {name}) if not format: self._group_index += 1 - return wrap % '.+?' + return wrap % r'.+?' # decode the format specification format = extract_format(format, self._extra_types) @@ -960,19 +970,19 @@ class Parser(object): return type_converter(string) self._type_conversions[group] = f elif type == 'n': - s = '\d{1,3}([,.]\d{3})*' + s = r'\d{1,3}([,.]\d{3})*' self._group_index += 1 self._type_conversions[group] = int_convert(10) elif type == 'b': - s = '(0[bB])?[01]+' + s = r'(0[bB])?[01]+' self._type_conversions[group] = int_convert(2) self._group_index += 1 elif type == 'o': - s = '(0[oO])?[0-7]+' + s = r'(0[oO])?[0-7]+' self._type_conversions[group] = int_convert(8) self._group_index += 1 elif type == 'x': - s = '(0[xX])?[0-9a-fA-F]+' + s = r'(0[xX])?[0-9a-fA-F]+' self._type_conversions[group] = int_convert(16) self._group_index += 1 elif type == '%': @@ -994,10 +1004,10 @@ class Parser(object): self._type_conversions[group] = lambda s, m: float(s) elif type == 'd': if format.get('width'): - width = '{1,%s}' % int(format['width']) + width = r'{1,%s}' % int(format['width']) else: width = '+' - s = '\\d{w}|0[xX][0-9a-fA-F]{w}|0[bB][01]{w}|0[oO][0-7]{w}'.format(w=width) + s = r'\d{w}|0[xX][0-9a-fA-F]{w}|0[bB][01]{w}|0[oO][0-7]{w}'.format(w=width) self._type_conversions[group] = int_convert(10) elif type == 'ti': s = r'(\d{4}-\d\d-\d\d)((\s+|T)%s)?(Z|\s*[-+]\d\d:?\d\d)?' % \ @@ -1055,18 +1065,19 @@ class Parser(object): self._type_conversions[group] = partial(date_convert, mm=n+1, dd=n+3, hms=n + 5) self._group_index += 5 - + elif type == 'l': + s = r'[A-Za-z]+' elif type: s = r'\%s+' % type elif format.get('precision'): if format.get('width'): - s = '.{%s,%s}?' % (format['width'], format['precision']) + s = r'.{%s,%s}?' % (format['width'], format['precision']) else: - s = '.{1,%s}?' % format['precision'] + s = r'.{1,%s}?' % format['precision'] elif format.get('width'): - s = '.{%s,}?' % format['width'] + s = r'.{%s,}?' % format['width'] else: - s = '.+?' + s = r'.+?' align = format['align'] fill = format['fill'] @@ -1079,7 +1090,7 @@ class Parser(object): # configurable fill defaulting to "0" if not fill: fill = '0' - s = '%s*' % fill + s + s = r'%s*' % fill + s # allow numbers to be prefixed with a sign s = r'[-+ ]?' + s @@ -1101,7 +1112,7 @@ class Parser(object): if not align: align = '>' - if fill in '.\+?*[](){}^$': + if fill in r'.\+?*[](){}^$': fill = '\\' + fill # align "=" has been handled @@ -1118,8 +1129,11 @@ class Parser(object): class Result(object): '''The result of a parse() or search(). - Fixed results may be looked up using result[index]. Named results may be - looked up using result['name']. + Fixed results may be looked up using `result[index]`. + + Named results may be looked up using `result['name']`. + + Named results may be tested for existence using `'name' in result`. ''' def __init__(self, fixed, named, spans): self.fixed = fixed @@ -1135,6 +1149,9 @@ class Result(object): return '<%s %r %r>' % (self.__class__.__name__, self.fixed, self.named) + def __contains__(self, name): + return name in self.named + class Match(object): '''The result of a parse() or search() if no results are generated. @@ -1295,7 +1312,7 @@ def compile(format, extra_types=None, case_sensitive=False): return Parser(format, extra_types=extra_types) -# Copyright (c) 2012-2013 Richard Jones +# Copyright (c) 2012-2019 Richard Jones # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/pipenv/vendor/pipdeptree.py b/pipenv/vendor/pipdeptree.py index cc15c24a..899118cc 100644 --- a/pipenv/vendor/pipdeptree.py +++ b/pipenv/vendor/pipdeptree.py @@ -22,7 +22,7 @@ import pkg_resources # from graphviz import backend, Digraph -__version__ = '0.13.1' +__version__ = '0.13.2' flatten = chain.from_iterable @@ -127,6 +127,13 @@ def guess_version(pkg_key, default='?'): return getattr(m, '__version__', default) +def frozen_req_from_dist(dist): + try: + return FrozenRequirement.from_dist(dist) + except TypeError: + return FrozenRequirement.from_dist(dist, []) + + class Package(object): """Abstract class for wrappers around objects that pip returns. @@ -154,7 +161,7 @@ class Package(object): @staticmethod def frozen_repr(obj): - fr = FrozenRequirement.from_dist(obj, []) + fr = frozen_req_from_dist(obj) return str(fr).strip() def __getattr__(self, key): @@ -563,7 +570,7 @@ def _get_args(): def main(): args = _get_args() pkgs = get_installed_distributions(local_only=args.local_only, - user_only=args.user_only) + user_only=args.user_only) dist_index = build_dist_index(pkgs) tree = construct_tree(dist_index) diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 9421573d..f3a981bd 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,16 +1,19 @@ from __future__ import absolute_import, print_function -__version__ = '1.1.11' - # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: # # > No handlers could be found for logger "pythonfinder.models.pyenv" import logging + +from .exceptions import InvalidPythonVersion +from .models import SystemPath, WindowsFinder +from .pythonfinder import Finder + +__version__ = "1.2.0" + + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) __all__ = ["Finder", "WindowsFinder", "SystemPath", "InvalidPythonVersion"] -from .pythonfinder import Finder -from .models import SystemPath, WindowsFinder -from .exceptions import InvalidPythonVersion diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index 7dc4ebd4..eb2e603a 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -13,9 +13,7 @@ from .pythonfinder import Finder @click.command() @click.option("--find", default=False, nargs=1, help="Find a specific python version.") @click.option("--which", default=False, nargs=1, help="Run the which command.") -@click.option( - "--findall", is_flag=True, default=False, help="Find all python versions." -) +@click.option("--findall", is_flag=True, default=False, help="Find all python versions.") @click.option( "--version", is_flag=True, default=False, help="Display PythonFinder version." ) diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index ce21fb79..eb000438 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -36,6 +36,7 @@ else: IGNORE_UNSUPPORTED = bool(os.environ.get("PYTHONFINDER_IGNORE_UNSUPPORTED", False)) MYPY_RUNNING = os.environ.get("MYPY_RUNNING", is_type_checking()) + def get_shim_paths(): shim_paths = [] if ASDF_INSTALLED: @@ -43,4 +44,6 @@ def get_shim_paths(): if PYENV_INSTALLED: shim_paths.append(os.path.join(PYENV_ROOT, "shims")) return [os.path.normpath(os.path.normcase(p)) for p in shim_paths] + + SHIM_PATHS = get_shim_paths() diff --git a/pipenv/vendor/pythonfinder/models/mixins.py b/pipenv/vendor/pythonfinder/models/mixins.py index 31c3c202..c1e7312a 100644 --- a/pipenv/vendor/pythonfinder/models/mixins.py +++ b/pipenv/vendor/pythonfinder/models/mixins.py @@ -3,23 +3,23 @@ from __future__ import absolute_import, unicode_literals import abc import operator - from collections import defaultdict import attr import six - from cached_property import cached_property from vistir.compat import fs_str from ..environment import MYPY_RUNNING from ..exceptions import InvalidPythonVersion from ..utils import ( - KNOWN_EXTS, Sequence, expand_paths, looks_like_python, - path_is_known_executable + KNOWN_EXTS, + Sequence, + expand_paths, + looks_like_python, + path_is_known_executable, ) - if MYPY_RUNNING: from .path import PathEntry from .python import PythonVersion @@ -48,7 +48,9 @@ class BasePath(object): only_python = attr.ib(default=False) # type: bool name = attr.ib(type=str) _py_version = attr.ib(default=None) # type: Optional[PythonVersion] - _pythons = attr.ib(default=attr.Factory(defaultdict)) # type: DefaultDict[str, PathEntry] + _pythons = attr.ib( + default=attr.Factory(defaultdict) + ) # type: DefaultDict[str, PathEntry] def __str__(self): # type: () -> str @@ -197,6 +199,7 @@ class BasePath(object): # type: () -> DefaultDict[Union[str, Path], PathEntry] if not self._pythons: from .path import PathEntry + self._pythons = defaultdict(PathEntry) for python in self._iter_pythons(): python_path = python.path.as_posix() # type: ignore @@ -241,17 +244,13 @@ class BasePath(object): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - call_method = ( - "find_all_python_versions" if self.is_dir else "find_python_version" - ) + call_method = "find_all_python_versions" if self.is_dir else "find_python_version" sub_finder = operator.methodcaller( call_method, major, minor, patch, pre, dev, arch, name ) if not self.is_dir: return sub_finder(self) - unnested = [ - sub_finder(path) for path in expand_paths(self) - ] + unnested = [sub_finder(path) for path in expand_paths(self)] version_sort = operator.attrgetter("as_python.version_sort") unnested = [p for p in unnested if p is not None and p.as_python is not None] paths = sorted(unnested, key=version_sort, reverse=True) @@ -291,13 +290,13 @@ class BasePath(object): matching_pythons = [ [entry, entry.as_python.version_sort] for entry in self._iter_pythons() - if (entry is not None and entry.as_python is not None and - version_matcher(entry.py_version)) + if ( + entry is not None + and entry.as_python is not None + and version_matcher(entry.py_version) + ) ] - results = sorted(matching_pythons, - key=operator.itemgetter(1, 0), - reverse=True, - ) + results = sorted(matching_pythons, key=operator.itemgetter(1, 0), reverse=True) return next(iter(r[0] for r in results if r is not None), None) @@ -316,9 +315,8 @@ class BaseFinder(object): raise NotImplementedError @classmethod - def create(cls, # type: Type[BaseFinderType] - *args, # type: Any - **kwargs # type: Any + def create( + cls, *args, **kwargs # type: Type[BaseFinderType] # type: Any # type: Any ): # type: (...) -> BaseFinderType raise NotImplementedError diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 005a17b1..9e099b59 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -5,60 +5,90 @@ import copy import operator import os import sys - from collections import defaultdict from itertools import chain import attr import six - from cached_property import cached_property from vistir.compat import Path, fs_str +from .mixins import BaseFinder, BasePath +from .python import PythonVersion from ..environment import ( - ASDF_DATA_DIR, ASDF_INSTALLED, MYPY_RUNNING, PYENV_INSTALLED, PYENV_ROOT, - SHIM_PATHS + ASDF_DATA_DIR, + ASDF_INSTALLED, + MYPY_RUNNING, + PYENV_INSTALLED, + PYENV_ROOT, + SHIM_PATHS, ) from ..exceptions import InvalidPythonVersion from ..utils import ( - Iterable, Sequence, ensure_path, expand_paths, filter_pythons, is_in_path, - looks_like_python, normalize_path, optional_instance_of, - parse_asdf_version_order, parse_pyenv_version_order, - path_is_known_executable, unnest + Iterable, + Sequence, + ensure_path, + expand_paths, + filter_pythons, + is_in_path, + looks_like_python, + normalize_path, + optional_instance_of, + parse_asdf_version_order, + parse_pyenv_version_order, + path_is_known_executable, + unnest, ) -from .mixins import BaseFinder, BasePath -from .python import PythonVersion - if MYPY_RUNNING: from typing import ( - Optional, Dict, DefaultDict, Iterator, List, Union, Tuple, Generator, Callable, - Type, Any, TypeVar + Optional, + Dict, + DefaultDict, + Iterator, + List, + Union, + Tuple, + Generator, + Callable, + Type, + Any, + TypeVar, ) - from .mixins import BaseFinder from .python import PythonFinder from .windows import WindowsFinder - FinderType = TypeVar('FinderType', BaseFinder, PythonFinder, WindowsFinder) - ChildType = Union[PythonFinder, PathEntry] - PathType = Union[PythonFinder, PathEntry] + + FinderType = TypeVar("FinderType", BaseFinder, PythonFinder, WindowsFinder) + ChildType = Union[PythonFinder, "PathEntry"] + PathType = Union[PythonFinder, "PathEntry"] @attr.s class SystemPath(object): global_search = attr.ib(default=True) - paths = attr.ib(default=attr.Factory(defaultdict)) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] + paths = attr.ib( + default=attr.Factory(defaultdict) + ) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] _executables = attr.ib(default=attr.Factory(list)) # type: List[PathEntry] - _python_executables = attr.ib(default=attr.Factory(dict)) # type: Dict[str, PathEntry] + _python_executables = attr.ib( + default=attr.Factory(dict) + ) # type: Dict[str, PathEntry] path_order = attr.ib(default=attr.Factory(list)) # type: List[str] python_version_dict = attr.ib() # type: DefaultDict[Tuple, List[PythonVersion]] only_python = attr.ib(default=False, type=bool) - pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PythonFinder")) # type: Optional[PythonFinder] + pyenv_finder = attr.ib( + default=None, validator=optional_instance_of("PythonFinder") + ) # type: Optional[PythonFinder] asdf_finder = attr.ib(default=None) # type: Optional[PythonFinder] system = attr.ib(default=False, type=bool) - _version_dict = attr.ib(default=attr.Factory(defaultdict)) # type: DefaultDict[Tuple, List[PathEntry]] + _version_dict = attr.ib( + default=attr.Factory(defaultdict) + ) # type: DefaultDict[Tuple, List[PathEntry]] ignore_unsupported = attr.ib(default=False, type=bool) - __finders = attr.ib(default=attr.Factory(dict)) # type: Dict[str, Union[WindowsFinder, PythonFinder]] + __finders = attr.ib( + default=attr.Factory(dict) + ) # type: Dict[str, Union[WindowsFinder, PythonFinder]] def _register_finder(self, finder_name, finder): # type: (str, Union[WindowsFinder, PythonFinder]) -> None @@ -117,7 +147,9 @@ class SystemPath(object): @cached_property def version_dict(self): # type: () -> DefaultDict[Tuple, List[PathEntry]] - self._version_dict = defaultdict(list) # type: DefaultDict[Tuple, List[PathEntry]] + self._version_dict = defaultdict( + list + ) # type: DefaultDict[Tuple, List[PathEntry]] for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): if finder_name == "windows": @@ -171,9 +203,7 @@ class SystemPath(object): reversed_paths = reversed(self.path_order) paths = [normalize_path(p) for p in reversed_paths] normalized_target = normalize_path(path) - last_instance = next( - iter(p for p in paths if normalized_target in p), None - ) + last_instance = next(iter(p for p in paths if normalized_target in p), None) if last_instance is None: raise ValueError("No instance found on path for target: {0!s}".format(path)) path_index = self.path_order.index(last_instance) @@ -190,19 +220,14 @@ class SystemPath(object): else: before_path = self.path_order[: start_idx + 1] after_path = self.path_order[start_idx + 2 :] - self.path_order = ( - before_path + [p.as_posix() for p in paths] + after_path - ) + self.path_order = before_path + [p.as_posix() for p in paths] + after_path def _remove_path(self, path): # type: (str) -> None path_copy = [p for p in reversed(self.path_order[:])] new_order = [] target = normalize_path(path) - path_map = { - normalize_path(pth): pth - for pth in self.paths.keys() - } + path_map = {normalize_path(pth): pth for pth in self.paths.keys()} if target in path_map: del self.paths[path_map[target]] for current_path in path_copy: @@ -215,10 +240,14 @@ class SystemPath(object): def _setup_asdf(self): # type: () -> None from .python import PythonFinder + os_path = os.environ["PATH"].split(os.pathsep) self.asdf_finder = PythonFinder.create( - root=ASDF_DATA_DIR, ignore_unsupported=True, - sort_function=parse_asdf_version_order, version_glob_path="installs/python/*") + root=ASDF_DATA_DIR, + ignore_unsupported=True, + sort_function=parse_asdf_version_order, + version_glob_path="installs/python/*", + ) asdf_index = None try: asdf_index = self._get_last_instance(ASDF_DATA_DIR) @@ -253,7 +282,9 @@ class SystemPath(object): # TODO: This is called 'reload', should we load a new finder for the first # time here? lets just skip that for now to avoid unallowed finders pass - if (finder_name == "pyenv" and not PYENV_INSTALLED) or (finder_name == "asdf" and not ASDF_INSTALLED): + if (finder_name == "pyenv" and not PYENV_INSTALLED) or ( + finder_name == "asdf" and not ASDF_INSTALLED + ): # Don't allow loading of finders that aren't explicitly 'installed' as it were pass setattr(self, finder_attr, None) @@ -264,10 +295,15 @@ class SystemPath(object): def _setup_pyenv(self): # type: () -> None from .python import PythonFinder + os_path = os.environ["PATH"].split(os.pathsep) self.pyenv_finder = PythonFinder.create( - root=PYENV_ROOT, sort_function=parse_pyenv_version_order, version_glob_path="versions/*", ignore_unsupported=self.ignore_unsupported) + root=PYENV_ROOT, + sort_function=parse_pyenv_version_order, + version_glob_path="versions/*", + ignore_unsupported=self.ignore_unsupported, + ) pyenv_index = None try: pyenv_index = self._get_last_instance(PYENV_ROOT) @@ -469,8 +505,7 @@ class SystemPath(object): name = "{0!s}".format(major) major = None sub_finder = operator.methodcaller( - "find_python_version", - major, minor, patch, pre, dev, arch, name, + "find_python_version", major, minor, patch, pre, dev, arch, name ) alternate_sub_finder = None if name and not (minor or patch or pre or dev or arch or major): @@ -517,7 +552,9 @@ class SystemPath(object): :rtype: :class:`pythonfinder.models.SystemPath` """ - path_entries = defaultdict(PathEntry) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] + path_entries = defaultdict( + PathEntry + ) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] paths = [] # type: List[str] if ignore_unsupported: os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1") @@ -567,6 +604,7 @@ class PathEntry(BasePath): def _gen_children(self): # type: () -> Iterator from ..environment import get_shim_paths + shim_paths = get_shim_paths() pass_name = self.name != self.path.name pass_args = {"is_root": False, "only_python": self.only_python} @@ -602,7 +640,6 @@ class PathEntry(BasePath): self._children = children return self._children - @classmethod def create(cls, path, is_root=False, only_python=False, pythons=None, name=None): # type: (Union[str, Path], bool, bool, Dict[str, PythonVersion], Optional[str]) -> PathEntry @@ -622,16 +659,18 @@ class PathEntry(BasePath): if not name: guessed_name = True name = target.name - creation_args = {"path": target, "is_root": is_root, "only_python": only_python, "name": name} + creation_args = { + "path": target, + "is_root": is_root, + "only_python": only_python, + "name": name, + } if pythons: creation_args["pythons"] = pythons _new = cls(**creation_args) if pythons and only_python: children = {} - child_creation_args = { - "is_root": False, - "only_python": only_python - } + child_creation_args = {"is_root": False, "only_python": only_python} if not guessed_name: child_creation_args["name"] = _new.name # type: ignore for pth, python in pythons.items(): @@ -639,9 +678,7 @@ class PathEntry(BasePath): continue pth = ensure_path(pth) children[pth.as_posix()] = PathEntry( # type: ignore - py_version=python, - path=pth, - **child_creation_args + py_version=python, path=pth, **child_creation_args ) _new._children = children return _new @@ -658,6 +695,7 @@ class VersionPath(SystemPath): Generates the version listings for it""" from .path import PathEntry + path = ensure_path(path) path_entries = defaultdict(PathEntry) bin_ = "{base}/bin" diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 8900660e..25a12d66 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -6,29 +6,44 @@ import logging import operator import platform import sys - from collections import defaultdict import attr import six - from packaging.version import Version from vistir.compat import Path, lru_cache +from .mixins import BaseFinder, BasePath from ..environment import ASDF_DATA_DIR, MYPY_RUNNING, PYENV_ROOT, SYSTEM_ARCH from ..exceptions import InvalidPythonVersion from ..utils import ( - RE_MATCHER, _filter_none, ensure_path, get_python_version, is_in_path, - looks_like_python, optional_instance_of, parse_asdf_version_order, - parse_pyenv_version_order, parse_python_version, unnest + RE_MATCHER, + _filter_none, + ensure_path, + get_python_version, + is_in_path, + looks_like_python, + optional_instance_of, + parse_asdf_version_order, + parse_pyenv_version_order, + parse_python_version, + unnest, ) -from .mixins import BaseFinder, BasePath - if MYPY_RUNNING: from typing import ( - DefaultDict, Optional, Callable, Generator, Any, Union, Tuple, List, Dict, Type, - TypeVar, Iterator + DefaultDict, + Optional, + Callable, + Generator, + Any, + Union, + Tuple, + List, + Dict, + Type, + TypeVar, + Iterator, ) from .path import PathEntry from .._vendor.pep514tools.environment import Environment @@ -68,8 +83,7 @@ class PythonFinder(BaseFinder, BasePath): def expanded_paths(self): # type: () -> Generator return ( - path for path in unnest(p for p in self.versions.values()) - if path is not None + path for path in unnest(p for p in self.versions.values()) if path is not None ) @property @@ -85,15 +99,20 @@ class PythonFinder(BaseFinder, BasePath): def get_version_order(self): # type: () -> List[Path] version_paths = [ - p for p in self.root.glob(self.version_glob_path) + p + for p in self.root.glob(self.version_glob_path) if not (p.parent.name == "envs" or p.name == "envs") ] versions = {v.name: v for v in version_paths} version_order = [] # type: List[Path] if self.is_pyenv: - version_order = [versions[v] for v in parse_pyenv_version_order() if v in versions] + version_order = [ + versions[v] for v in parse_pyenv_version_order() if v in versions + ] elif self.is_asdf: - version_order = [versions[v] for v in parse_asdf_version_order() if v in versions] + version_order = [ + versions[v] for v in parse_asdf_version_order() if v in versions + ] for version in version_order: version_paths.remove(version) if version_order: @@ -118,12 +137,12 @@ class PythonFinder(BaseFinder, BasePath): def _iter_version_bases(self): # type: () -> Iterator[Tuple[Path, PathEntry]] from .path import PathEntry + for p in self.get_version_order(): bin_dir = self.get_bin_dir(p) if bin_dir.exists() and bin_dir.is_dir(): entry = PathEntry.create( - path=bin_dir.absolute(), only_python=False, name=p.name, - is_root=True + path=bin_dir.absolute(), only_python=False, name=p.name, is_root=True ) self.roots[p] = entry yield (p, entry) @@ -146,8 +165,11 @@ class PythonFinder(BaseFinder, BasePath): except Exception: if not self.ignore_unsupported: raise - logger.warning("Unsupported Python version %r, ignoring...", - base_path.name, exc_info=True) + logger.warning( + "Unsupported Python version %r, ignoring...", + base_path.name, + exc_info=True, + ) continue if version is not None: version_tuple = ( @@ -190,6 +212,7 @@ class PythonFinder(BaseFinder, BasePath): # type: () -> DefaultDict[str, PathEntry] if not self._pythons: from .path import PathEntry + self._pythons = defaultdict(PathEntry) # type: DefaultDict[str, PathEntry] for python in self._iter_pythons(): python_path = python.path.as_posix() # type: ignore @@ -206,13 +229,20 @@ class PythonFinder(BaseFinder, BasePath): return self.pythons @classmethod - def create(cls, root, sort_function, version_glob_path=None, ignore_unsupported=True): # type: ignore + def create( + cls, root, sort_function, version_glob_path=None, ignore_unsupported=True + ): # type: ignore # type: (Type[PythonFinder], str, Callable, Optional[str], bool) -> PythonFinder root = ensure_path(root) if not version_glob_path: version_glob_path = "versions/*" - return cls(root=root, path=root, ignore_unsupported=ignore_unsupported, # type: ignore - sort_function=sort_function, version_glob_path=version_glob_path) + return cls( + root=root, + path=root, + ignore_unsupported=ignore_unsupported, # type: ignore + sort_function=sort_function, + version_glob_path=version_glob_path, + ) def find_all_python_versions( self, @@ -239,9 +269,7 @@ class PythonFinder(BaseFinder, BasePath): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - call_method = ( - "find_all_python_versions" if self.is_dir else "find_python_version" - ) + call_method = "find_all_python_versions" if self.is_dir else "find_python_version" sub_finder = operator.methodcaller( call_method, major, minor, patch, pre, dev, arch, name ) @@ -251,13 +279,12 @@ class PythonFinder(BaseFinder, BasePath): for _, base in self._iter_version_bases() ] else: - pythons = [ - sub_finder(path) for path in self.paths - ] + pythons = [sub_finder(path) for path in self.paths] pythons = [p for p in pythons if p and p.is_python and p.as_python is not None] version_sort = operator.attrgetter("as_python.version_sort") paths = [ - p for p in sorted(list(pythons), key=version_sort, reverse=True) + p + for p in sorted(list(pythons), key=version_sort, reverse=True) if p is not None ] return paths @@ -292,7 +319,8 @@ class PythonFinder(BaseFinder, BasePath): version_sort = operator.attrgetter("as_python.version_sort") unnested = [sub_finder(self.roots[path]) for path in self.roots] unnested = [ - p for p in unnested + p + for p in unnested if p is not None and p.is_python and p.as_python is not None ] paths = sorted(list(unnested), key=version_sort, reverse=True) @@ -527,6 +555,7 @@ class PythonVersion(object): if not isinstance(path, PathEntry): path = PathEntry.create(path, is_root=False, only_python=True, name=name) from ..environment import IGNORE_UNSUPPORTED + ignore_unsupported = ignore_unsupported or IGNORE_UNSUPPORTED path_name = getattr(path, "name", path.path.name) # str if not path.is_python: @@ -540,7 +569,10 @@ class PythonVersion(object): if instance_dict.get("minor") is None and looks_like_python(path.path.name): instance_dict = cls.parse_executable(path.path.absolute().as_posix()) - if not isinstance(instance_dict.get("version"), Version) and not ignore_unsupported: + if ( + not isinstance(instance_dict.get("version"), Version) + and not ignore_unsupported + ): raise ValueError("Not a valid python path: %s" % path) if instance_dict.get("patch") is None: instance_dict = cls.parse_executable(path.path.absolute().as_posix()) @@ -596,7 +628,7 @@ class PythonVersion(object): launcher_entry.info, "sys_architecture", SYSTEM_ARCH ), "executable": exe_path, - "name": name + "name": name, } ) py_version = cls.create(**creation_dict) @@ -616,7 +648,9 @@ class PythonVersion(object): @attr.s class VersionMap(object): - versions = attr.ib(factory=defaultdict) # type: DefaultDict[Tuple[int, Optional[int], Optional[int], bool, bool, bool], List[PathEntry]] + versions = attr.ib( + factory=defaultdict + ) # type: DefaultDict[Tuple[int, Optional[int], Optional[int], bool, bool, bool], List[PathEntry]] def add_entry(self, entry): # type: (...) -> None @@ -634,8 +668,8 @@ class VersionMap(object): self.versions[version] = entries else: current_entries = { - p.path for p in - self.versions[version] # type: ignore + p.path + for p in self.versions[version] # type: ignore if version in self.versions } new_entries = {p.path for p in entries} diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index 5d446339..e96e1081 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -2,22 +2,21 @@ from __future__ import absolute_import, print_function import operator - from collections import defaultdict import attr -from ..environment import MYPY_RUNNING -from ..exceptions import InvalidPythonVersion -from ..utils import ensure_path from .mixins import BaseFinder from .path import PathEntry from .python import PythonVersion, VersionMap - +from ..environment import MYPY_RUNNING +from ..exceptions import InvalidPythonVersion +from ..utils import ensure_path if MYPY_RUNNING: from typing import DefaultDict, Tuple, List, Optional, Union, TypeVar, Type, Any - FinderType = TypeVar('FinderType') + + FinderType = TypeVar("FinderType") @attr.s @@ -41,9 +40,7 @@ class WindowsFinder(BaseFinder): version_matcher = operator.methodcaller( "matches", major, minor, patch, pre, dev, arch, python_name=name ) - pythons = [ - py for py in self.version_list if version_matcher(py) - ] + pythons = [py for py in self.version_list if version_matcher(py)] version_sort = operator.attrgetter("version_sort") return [c.comes_from for c in sorted(pythons, key=version_sort, reverse=True)] @@ -58,15 +55,20 @@ class WindowsFinder(BaseFinder): name=None, # type: Optional[str] ): # type: (...) -> Optional[PathEntry] - return next(iter(v for v in self.find_all_python_versions( - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, - )), None + return next( + iter( + v + for v in self.find_all_python_versions( + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + ) + ), + None, ) @_versions.default @@ -92,13 +94,14 @@ class WindowsFinder(BaseFinder): if py_version is None: continue self.version_list.append(py_version) - python_path = py_version.comes_from.path if py_version.comes_from else py_version.executable + python_path = ( + py_version.comes_from.path + if py_version.comes_from + else py_version.executable + ) python_kwargs = {python_path: py_version} if python_path is not None else {} base_dir = PathEntry.create( - path, - is_root=True, - only_python=True, - pythons=python_kwargs, + path, is_root=True, only_python=True, pythons=python_kwargs ) versions[py_version.version_tuple[:5]] = base_dir self.paths.append(base_dir) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 63f63c74..d5f38fb4 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -5,16 +5,14 @@ import operator import os import six - from click import secho from vistir.compat import lru_cache from . import environment from .exceptions import InvalidPythonVersion -from .models import path +from .models import path as pyfinder_path from .utils import Iterable, filter_pythons, version_re - if environment.MYPY_RUNNING: from typing import Optional, Dict, Any, Union, List, Iterator from .models.path import Path, PathEntry @@ -34,7 +32,9 @@ class Finder(object): *path* and *system*. """ - def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=True): + def __init__( + self, path=None, system=False, global_search=True, ignore_unsupported=True + ): # type: (Optional[str], bool, bool, bool) -> None """Create a new :class:`~pythonfinder.pythonfinder.Finder` instance. @@ -68,9 +68,11 @@ class Finder(object): def create_system_path(self): # type: () -> SystemPath - return path.SystemPath.create( - path=self.path_prepend, system=self.system, global_search=self.global_search, - ignore_unsupported=self.ignore_unsupported + return pyfinder_path.SystemPath.create( + path=self.path_prepend, + system=self.system, + global_search=self.global_search, + ignore_unsupported=self.ignore_unsupported, ) def reload_system_path(self): @@ -84,7 +86,7 @@ class Finder(object): if self._system_path is not None: self._system_path.clear_caches() self._system_path = None - six.moves.reload_module(path) + six.moves.reload_module(pyfinder_path) self._system_path = self.create_system_path() def rehash(self): @@ -136,12 +138,13 @@ class Finder(object): """ from .models import PythonVersion + minor = int(minor) if minor is not None else minor patch = int(patch) if patch is not None else patch version_dict = { "minor": minor, - "patch": patch + "patch": patch, } # type: Dict[str, Union[str, int, Any]] if ( @@ -177,7 +180,9 @@ class Finder(object): if "." in major and all(part.isdigit() for part in major.split(".")[:2]): match = version_re.match(major) version_dict = match.groupdict() - version_dict["is_prerelease"] = bool(version_dict.get("prerel", False)) + version_dict["is_prerelease"] = bool( + version_dict.get("prerel", False) + ) version_dict["is_devrelease"] = bool(version_dict.get("dev", False)) else: version_dict = { @@ -186,7 +191,7 @@ class Finder(object): "patch": patch, "pre": pre, "dev": dev, - "arch": arch + "arch": arch, } if version_dict.get("minor") is not None: minor = int(version_dict["minor"]) @@ -198,10 +203,18 @@ class Finder(object): pre = bool(_pre) if _pre is not None else pre _dev = version_dict.get("is_devrelease", dev) dev = bool(_dev) if _dev is not None else dev - arch = version_dict.get("architecture", None) if arch is None else arch # type: ignore + arch = ( + version_dict.get("architecture", None) if arch is None else arch + ) # type: ignore if os.name == "nt" and self.windows_finder is not None: match = self.windows_finder.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, ) if match: return match @@ -218,10 +231,10 @@ class Finder(object): python_version_dict = getattr(self.system_path, "python_version_dict") if python_version_dict: paths = ( - path - for version in python_version_dict.values() - for path in version - if path is not None and path.as_python + path + for version in python_version_dict.values() + for path in version + if path is not None and path.as_python ) path_list = sorted(paths, key=version_sort, reverse=True) return path_list @@ -229,7 +242,7 @@ class Finder(object): major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) if not isinstance(versions, Iterable): - versions = [versions,] + versions = [versions] path_list = sorted(versions, key=version_sort, reverse=True) path_map = {} # type: Dict[str, PathEntry] for path in path_list: diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 24ffdea0..a82654f3 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -5,23 +5,26 @@ import io import itertools import os import re - from fnmatch import fnmatch import attr import six import vistir - from packaging.version import LegacyVersion, Version from .environment import MYPY_RUNNING, PYENV_ROOT from .exceptions import InvalidPythonVersion - -six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) # type: ignore # noqa -six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) # type: ignore # noqa -from six.moves import Iterable # type: ignore # noqa -from six.moves import Sequence # type: ignore # noqa +six.add_move( + six.MovedAttribute("Iterable", "collections", "collections.abc") +) # type: ignore # noqa +six.add_move( + six.MovedAttribute("Sequence", "collections", "collections.abc") +) # type: ignore # noqa +# fmt: off +from six.moves import Iterable # type: ignore # noqa # isort:skip +from six.moves import Sequence # type: ignore # noqa # isort:skip +# fmt: on try: from functools import lru_cache @@ -29,27 +32,42 @@ except ImportError: from backports.functools_lru_cache import lru_cache # type: ignore # noqa if MYPY_RUNNING: - from typing import ( - Any, Union, List, Callable, Iterable, Set, Tuple, Dict, Optional, Iterator - ) + from typing import Any, Union, List, Callable, Set, Tuple, Dict, Optional, Iterator from attr.validators import _OptionalValidator # type: ignore from .models.path import PathEntry -version_re = re.compile(r"(?P\d+)(?:\.(?P\d+))?(?:\.(?P(?<=\.)[0-9]+))?\.?" - r"(?:(?P[abc]|rc|dev)(?:(?P\d+(?:\.\d+)*))?)" - r"?(?P(\.post(?P\d+))?(\.dev(?P\d+))?)?") +version_re = re.compile( + r"(?P\d+)(?:\.(?P\d+))?(?:\.(?P(?<=\.)[0-9]+))?\.?" + r"(?:(?P[abc]|rc|dev)(?:(?P\d+(?:\.\d+)*))?)" + r"?(?P(\.post(?P\d+))?(\.dev(?P\d+))?)?" +) PYTHON_IMPLEMENTATIONS = ( - "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", - "stackless", "activepython", "micropython" + "python", + "ironpython", + "jython", + "pypy", + "anaconda", + "miniconda", + "stackless", + "activepython", + "micropython", +) +RE_MATCHER = re.compile( + r"(({0})(?:\d?(?:\.\d[cpm]{{0,3}}))?(?:-?[\d\.]+)*[^z])".format( + "|".join(PYTHON_IMPLEMENTATIONS) + ) ) -RE_MATCHER = re.compile(r"(({0})(?:\d?(?:\.\d[cpm]{{0,3}}))?(?:-?[\d\.]+)*[^z])".format( - "|".join(PYTHON_IMPLEMENTATIONS) -)) RULES_BASE = [ - "*{0}", "*{0}?", "*{0}?.?", "*{0}?.?m", "{0}?-?.?", "{0}?-?.?.?", "{0}?.?-?.?.?" + "*{0}", + "*{0}?", + "*{0}?.?", + "*{0}?.?m", + "{0}?-?.?", + "{0}?-?.?.?", + "{0}?.?-?.?.?", ] RULES = [rule.format(impl) for impl in PYTHON_IMPLEMENTATIONS for rule in RULES_BASE] @@ -61,10 +79,7 @@ KNOWN_EXTS = KNOWN_EXTS | set( MATCH_RULES = [] for rule in RULES: MATCH_RULES.extend( - [ - "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) - for ext in KNOWN_EXTS - ] + ["{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) for ext in KNOWN_EXTS] ) @@ -74,8 +89,14 @@ def get_python_version(path): """Get python version string using subprocess from a given path.""" version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: - c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, - combine_stderr=False, write_to_stdout=False) + c = vistir.misc.run( + version_cmd, + block=True, + nospin=True, + return_object=True, + combine_stderr=False, + write_to_stdout=False, + ) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) if not c.out: @@ -87,6 +108,7 @@ def get_python_version(path): def parse_python_version(version_str): # type: (str) -> Dict[str, Union[str, int, Version]] from packaging.version import parse as parse_version + is_debug = False if version_str.endswith("-debug"): is_debug = True @@ -127,7 +149,7 @@ def parse_python_version(version_str): "is_prerelease": is_prerelease, "is_devrelease": is_devrelease, "is_debug": is_debug, - "version": version + "version": version, } @@ -237,9 +259,11 @@ def _filter_none(k, v): # TODO: Reimplement in vistir def normalize_path(path): # type: (str) -> str - return os.path.normpath(os.path.normcase( - os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) - )) + return os.path.normpath( + os.path.normcase( + os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) + ) + ) @lru_cache(maxsize=1024) @@ -290,9 +314,10 @@ def parse_asdf_version_order(filename=".tool-versions"): if os.path.exists(version_order_file) and os.path.isfile(version_order_file): with io.open(version_order_file, encoding="utf-8") as fh: contents = fh.read() - python_section = next(iter( - line for line in contents.splitlines() if line.startswith("python") - ), None) + python_section = next( + iter(line for line in contents.splitlines() if line.startswith("python")), + None, + ) if python_section: # python_key, _, versions _, _, versions = python_section.partition(" ") @@ -317,8 +342,10 @@ def expand_paths(path, only_python=True): :rtype: Iterator[PathEntry] """ - if path is not None and (isinstance(path, Sequence) and - not getattr(path.__class__, "__name__", "") == "PathEntry"): + if path is not None and ( + isinstance(path, Sequence) + and not getattr(path.__class__, "__name__", "") == "PathEntry" + ): for p in unnest(path): if p is None: continue diff --git a/pipenv/vendor/pytoml/LICENSE b/pipenv/vendor/pytoml/LICENSE new file mode 100644 index 00000000..9739fc67 --- /dev/null +++ b/pipenv/vendor/pytoml/LICENSE @@ -0,0 +1,16 @@ +No-notice MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 54e71e8d..c3e237ac 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,17 +1,21 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function -__version__ = '1.4.2' import logging import warnings + from vistir.compat import ResourceWarning +from .models.lockfile import Lockfile +from .models.pipfile import Pipfile +from .models.requirements import Requirement + +__version__ = "1.4.2" + + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) warnings.filterwarnings("ignore", category=ResourceWarning) -from .models.requirements import Requirement -from .models.lockfile import Lockfile -from .models.pipfile import Pipfile __all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index f87fd585..44f34edb 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -20,6 +20,7 @@ from vistir.contextmanagers import cd, temp_environ from vistir.misc import partialclass from vistir.path import create_tracked_tempdir +from ..environment import MYPY_RUNNING from ..utils import prepare_pip_source_args, _ensure_dir from .cache import CACHE_DIR, DependencyCache from .utils import ( @@ -29,6 +30,17 @@ from .utils import ( ) +if MYPY_RUNNING: + from typing import Any, Dict, List, Generator, Optional, Union, Tuple, TypeVar, Text, Set, AnyStr + from pip_shims.shims import InstallRequirement, InstallationCandidate, PackageFinder, Command + from packaging.requirements import Requirement as PackagingRequirement + TRequirement = TypeVar("TRequirement") + RequirementType = TypeVar('RequirementType', covariant=True, bound=PackagingRequirement) + MarkerType = TypeVar('MarkerType', covariant=True, bound=Marker) + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) + + PKGS_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "pkgs")) WHEEL_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "wheels")) @@ -43,6 +55,7 @@ def _get_filtered_versions(ireq, versions, prereleases): def find_all_matches(finder, ireq, pre=False): + # type: (PackageFinder, InstallRequirement, bool) -> List[InstallationCandidate] """Find all matching dependencies using the supplied finder and the given ireq. @@ -65,6 +78,7 @@ def find_all_matches(finder, ireq, pre=False): def get_pip_command(): + # type: () -> Command # Use pip's parser for pip.conf management and defaults. # General options (find_links, index_url, extra_index_url, trusted_host, # and pre) are defered to pip. @@ -89,7 +103,7 @@ def get_pip_command(): @attr.s class AbstractDependency(object): - name = attr.ib() + name = attr.ib() # type: STRING_TYPE specifiers = attr.ib() markers = attr.ib() candidates = attr.ib() @@ -284,6 +298,7 @@ def get_abstract_dependencies(reqs, sources=None, parent=None): def get_dependencies(ireq, sources=None, parent=None): + # type: (Union[InstallRequirement, InstallationCandidate], Optional[List[Dict[S, Union[S, bool]]]], Optional[AbstractDependency]) -> Set[S, ...] """Get all dependencies for a given install requirement. :param ireq: A single InstallRequirement @@ -556,6 +571,7 @@ def get_pip_options(args=[], sources=None, pip_command=None): def get_finder(sources=None, pip_command=None, pip_options=None): + # type: (List[Dict[S, Union[S, bool]]], Optional[Command], Any) -> PackageFinder """Get a package finder for looking up candidates to install :param sources: A list of pipfile-formatted sources, defaults to None diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index 0aba3a43..515f9a88 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -132,7 +132,7 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): # type: (S) -> S - """Cleans VCS uris from pip format""" + """Cleans VCS uris from pipenv.patched.notpip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index ecc67c1a..5be3b747 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -3,40 +3,39 @@ from __future__ import absolute_import, unicode_literals from .compat import ( NamedTemporaryFile, + StringIO, TemporaryDirectory, partialmethod, to_native_string, - StringIO, ) from .contextmanagers import ( atomic_open_for_write, cd, open_file, + replaced_stream, + spinner, temp_environ, temp_path, - spinner, - replaced_stream ) -from .cursor import show_cursor, hide_cursor +from .cursor import hide_cursor, show_cursor from .misc import ( + StreamWrapper, + chunked, + decode_for_output, + divide, + get_wrapped_stream, load_path, partialclass, run, shell_escape, - decode_for_output, - to_text, - to_bytes, take, - chunked, - divide, - get_wrapped_stream, - StreamWrapper + to_bytes, + to_text, ) -from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile +from .path import create_tracked_tempdir, create_tracked_tempfile, mkdir_p, rmtree from .spin import create_spinner - -__version__ = '0.3.1' +__version__ = "0.3.1" __all__ = [ @@ -70,5 +69,5 @@ __all__ = [ "StreamWrapper", "replaced_stream", "show_cursor", - "hide_cursor" + "hide_cursor", ] diff --git a/pipenv/vendor/vistir/backports/__init__.py b/pipenv/vendor/vistir/backports/__init__.py index 0bdac1ea..03859e4f 100644 --- a/pipenv/vendor/vistir/backports/__init__.py +++ b/pipenv/vendor/vistir/backports/__init__.py @@ -4,8 +4,4 @@ from __future__ import absolute_import, unicode_literals from .functools import partialmethod from .tempfile import NamedTemporaryFile - -__all__ = [ - "NamedTemporaryFile", - "partialmethod" -] +__all__ = ["NamedTemporaryFile", "partialmethod"] diff --git a/pipenv/vendor/vistir/backports/functools.py b/pipenv/vendor/vistir/backports/functools.py index 8060d183..5d98f41a 100644 --- a/pipenv/vendor/vistir/backports/functools.py +++ b/pipenv/vendor/vistir/backports/functools.py @@ -3,8 +3,7 @@ from __future__ import absolute_import, unicode_literals from functools import partial - -__all__ = ["partialmethod",] +__all__ = ["partialmethod"] class partialmethod(object): @@ -16,8 +15,7 @@ class partialmethod(object): def __init__(self, func, *args, **keywords): if not callable(func) and not hasattr(func, "__get__"): - raise TypeError("{!r} is not callable or a descriptor" - .format(func)) + raise TypeError("{!r} is not callable or a descriptor".format(func)) # func could be a descriptor like classmethod which isn't callable, # so we can't inherit from partial (it verifies func is callable) @@ -36,26 +34,28 @@ class partialmethod(object): def __repr__(self): args = ", ".join(map(repr, self.args)) - keywords = ", ".join("{}={!r}".format(k, v) - for k, v in self.keywords.items()) + keywords = ", ".join("{}={!r}".format(k, v) for k, v in self.keywords.items()) format_string = "{module}.{cls}({func}, {args}, {keywords})" - return format_string.format(module=self.__class__.__module__, - cls=self.__class__.__qualname__, - func=self.func, - args=args, - keywords=keywords) + return format_string.format( + module=self.__class__.__module__, + cls=self.__class__.__qualname__, + func=self.func, + args=args, + keywords=keywords, + ) def _make_unbound_method(self): def _method(*args, **keywords): call_keywords = self.keywords.copy() call_keywords.update(keywords) if len(args) > 1: - cls_or_self, rest = args[0], tuple(args[1:],) + cls_or_self, rest = args[0], tuple(args[1:]) else: cls_or_self = args[0] rest = tuple() call_args = (cls_or_self,) + self.args + tuple(rest) return self.func(*call_args, **call_keywords) + _method.__isabstractmethod__ = self.__isabstractmethod__ _method._partialmethod = self return _method diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index fb044acf..ce66138b 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py @@ -5,7 +5,6 @@ import functools import io import os import sys - from tempfile import _bin_openflags, _mkstemp_inner, gettempdir import six @@ -175,7 +174,7 @@ def NamedTemporaryFile( prefix=None, dir=None, delete=True, - wrapper_class_override=None + wrapper_class_override=None, ): """Create and return a temporary file. Arguments: @@ -203,13 +202,11 @@ def NamedTemporaryFile( else: (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: - file = io.open( - fd, mode, buffering=buffering, newline=newline, encoding=encoding - ) + file = io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) if wrapper_class_override is not None: - return type( - str("_TempFileWrapper"), (wrapper_class_override, object), {} - )(file, name, delete) + return type(str("_TempFileWrapper"), (wrapper_class_override, object), {})( + file, name, delete + ) else: return _TemporaryFileWrapper(file, name, delete) diff --git a/pipenv/vendor/vistir/cmdparse.py b/pipenv/vendor/vistir/cmdparse.py index 5a307f24..664ae7df 100644 --- a/pipenv/vendor/vistir/cmdparse.py +++ b/pipenv/vendor/vistir/cmdparse.py @@ -7,7 +7,6 @@ import shlex import six - __all__ = ["ScriptEmptyError", "Script"] @@ -78,7 +77,9 @@ class Script(object): See also: https://docs.python.org/3/library/subprocess.html#converting-argument-sequence """ - return " ".join(itertools.chain( - [_quote_if_contains(self.command, r'[\s^()]')], - (_quote_if_contains(arg, r'[\s^]') for arg in self.args), - )) + return " ".join( + itertools.chain( + [_quote_if_contains(self.command, r"[\s^()]")], + (_quote_if_contains(arg, r"[\s^]") for arg in self.args), + ) + ) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index 260959e1..e8688d89 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -6,11 +6,11 @@ import errno import os import sys import warnings - from tempfile import mkdtemp import six +from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile __all__ = [ "Path", @@ -35,19 +35,20 @@ __all__ = [ "fs_encode", "fs_decode", "_fs_encode_errors", - "_fs_decode_errors" + "_fs_decode_errors", ] if sys.version_info >= (3, 5): from pathlib import Path from functools import lru_cache else: - from pathlib2 import Path + from pipenv.vendor.pathlib2 import Path from pipenv.vendor.backports.functools_lru_cache import lru_cache -from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile + if sys.version_info < (3, 3): from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + NamedTemporaryFile = _NamedTemporaryFile else: from tempfile import NamedTemporaryFile @@ -89,6 +90,7 @@ if six.PY2: class IsADirectoryError(OSError): """The command does not work on directories""" + pass class FileExistsError(OSError): @@ -96,19 +98,35 @@ if six.PY2: self.errno = errno.EEXIST super(FileExistsError, self).__init__(*args, **kwargs) + else: from builtins import ( - ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError, - FileExistsError + ResourceWarning, + FileNotFoundError, + PermissionError, + IsADirectoryError, + FileExistsError, ) from io import StringIO -six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) # type: ignore -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) # type: ignore -six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) # type: ignore +six.add_move( + six.MovedAttribute("Iterable", "collections", "collections.abc") +) # type: ignore +six.add_move( + six.MovedAttribute("Mapping", "collections", "collections.abc") +) # type: ignore +six.add_move( + six.MovedAttribute("Sequence", "collections", "collections.abc") +) # type: ignore six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # type: ignore -six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) # type: ignore -from six.moves import Iterable, Mapping, Sequence, Set, ItemsView # type: ignore # noqa +six.add_move( + six.MovedAttribute("ItemsView", "collections", "collections.abc") +) # type: ignore + +# fmt: off +from six.moves import ItemsView, Iterable, Mapping, Sequence, Set # type: ignore # noqa # isort:skip +# fmt: on + if not sys.warnoptions: warnings.simplefilter("default", ResourceWarning) @@ -282,6 +300,7 @@ else: def to_native_string(string): from .misc import to_text, to_bytes + if six.PY2: return to_bytes(string) return to_text(string) diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index b2627d2d..d9223b66 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -1,11 +1,10 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import absolute_import, print_function, unicode_literals import io import os import stat import sys - from contextlib import contextmanager import six @@ -13,10 +12,15 @@ import six from .compat import NamedTemporaryFile, Path from .path import is_file_url, is_valid_url, path_to_url, url_to_path - __all__ = [ - "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner", - "dummy_spinner", "replaced_stream" + "temp_environ", + "temp_path", + "cd", + "atomic_open_for_write", + "open_file", + "spinner", + "dummy_spinner", + "replaced_stream", ] @@ -104,7 +108,13 @@ def dummy_spinner(spin_type, text, **kwargs): @contextmanager -def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, write_to_stdout=True): +def spinner( + spinner_name=None, + start_text=None, + handler_map=None, + nospin=False, + write_to_stdout=True, +): """Get a spinner object or a dummy spinner to wrap a context. :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) @@ -120,6 +130,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, """ from .spin import create_spinner + has_yaspin = None try: import yaspin @@ -146,7 +157,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, handler_map=handler_map, nospin=nospin, use_yaspin=use_yaspin, - write_to_stdout=write_to_stdout + write_to_stdout=write_to_stdout, ) as _spinner: yield _spinner @@ -267,8 +278,8 @@ def open_file(link, session=None, stream=True): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(link)) else: - with io.open(local_path, "rb") as local_file: - yield local_file + with io.open(local_path, "rb") as local_file: + yield local_file else: # Remote URL headers = {"Accept-Encoding": "identity"} diff --git a/pipenv/vendor/vistir/cursor.py b/pipenv/vendor/vistir/cursor.py index ce998f51..22d643e1 100644 --- a/pipenv/vendor/vistir/cursor.py +++ b/pipenv/vendor/vistir/cursor.py @@ -5,13 +5,11 @@ import ctypes import os import sys - __all__ = ["hide_cursor", "show_cursor"] class CONSOLE_CURSOR_INFO(ctypes.Structure): - _fields_ = [('dwSize', ctypes.c_int), - ('bVisible', ctypes.c_int)] + _fields_ = [("dwSize", ctypes.c_int), ("bVisible", ctypes.c_int)] WIN_STDERR_HANDLE_ID = ctypes.c_ulong(-12) @@ -29,6 +27,7 @@ def get_stream_handle(stream=sys.stdout): handle = stream if os.name == "nt": from ctypes import windll + handle_id = WIN_STDOUT_HANDLE_ID handle = windll.kernel32.GetStdHandle(handle_id) return handle @@ -46,6 +45,7 @@ def hide_cursor(stream=sys.stdout): handle = get_stream_handle(stream=stream) if os.name == "nt": from ctypes import windll + cursor_info = CONSOLE_CURSOR_INFO() windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(cursor_info)) cursor_info.visible = False @@ -67,6 +67,7 @@ def show_cursor(stream=sys.stdout): handle = get_stream_handle(stream=stream) if os.name == "nt": from ctypes import windll + cursor_info = CONSOLE_CURSOR_INFO() windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(cursor_info)) cursor_info.visible = True diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index cf301233..42067b2d 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -1,14 +1,13 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import absolute_import, print_function, unicode_literals import io import json -import logging import locale +import logging import os import subprocess import sys - from collections import OrderedDict from functools import partial from itertools import islice, tee @@ -16,10 +15,11 @@ from itertools import islice, tee import six from .cmdparse import Script -from .compat import Path, fs_str, partialmethod, to_native_string, Iterable, StringIO +from .compat import Iterable, Path, StringIO, fs_str, partialmethod, to_native_string from .contextmanagers import spinner as spinner if os.name != "nt": + class WindowsError(OSError): pass @@ -144,6 +144,55 @@ def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=Tru return subprocess.Popen(script.cmdify(), **options) +def _read_streams(stream_dict): + results = {} + for outstream in stream_dict.keys(): + stream = stream_dict[outstream] + if not stream: + results[outstream] = None + continue + line = to_text(stream.readline()) + if not line: + results[outstream] = None + continue + line = to_text("{0}".format(line.rstrip())) + results[outstream] = line + return results + + +def get_stream_results(cmd_instance, verbose, maxlen, spinner=None, stdout_allowed=False): + stream_results = {"stdout": [], "stderr": []} + streams = {"stderr": cmd_instance.stderr, "stdout": cmd_instance.stdout} + while True: + stream_contents = _read_streams(streams) + stdout_line = stream_contents["stdout"] + stderr_line = stream_contents["stderr"] + if not (stdout_line or stderr_line): + break + for stream_name in stream_contents.keys(): + if stream_contents[stream_name] and stream_name in stream_results: + line = stream_contents[stream_name] + stream_results[stream_name].append(line) + display_line = fs_str("{0}".format(line)) + if len(display_line) > maxlen: + display_line = "{0}...".format(display_line[:maxlen]) + if verbose: + use_stderr = not stdout_allowed or stream_name != "stdout" + if spinner: + target = spinner.stderr if use_stderr else spinner.stdout + spinner.hide_and_write(display_line, target=target) + else: + target = sys.stderr if use_stderr else sys.stdout + target.write(display_line) + target.flush() + if spinner: + spinner.text = to_native_string( + "{0} {1}".format(spinner.text, display_line) + ) + continue + return stream_results + + def _create_subprocess( cmd, env=None, @@ -155,74 +204,35 @@ def _create_subprocess( combine_stderr=False, display_limit=200, start_text="", - write_to_stdout=True + write_to_stdout=True, ): if not env: env = os.environ.copy() try: - c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, - combine_stderr=combine_stderr) - except Exception as exc: + c = _spawn_subprocess( + cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr + ) + except Exception: import traceback + formatted_tb = "".join(traceback.format_exception(*sys.exc_info())) sys.stderr.write("Error while executing command %s:" % " ".join(cmd._parts)) sys.stderr.write(formatted_tb) raise if not block: c.stdin.close() - output = [] - err = [] - spinner_orig_text = None - if spinner: - spinner_orig_text = getattr(spinner, "text", None) - if spinner_orig_text is None: - spinner_orig_text = start_text if start_text is not None else "" - streams = { - "stdout": c.stdout, - "stderr": c.stderr - } - while True: - stdout_line = None - stderr_line = None - for outstream in streams.keys(): - stream = streams[outstream] - if not stream: - continue - line = to_text(stream.readline()) - if not line: - continue - line = to_text("{0}".format(line.rstrip())) - if outstream == "stderr": - stderr_line = line - else: - stdout_line = line - if not (stdout_line or stderr_line): - break - if stderr_line is not None: - err.append(stderr_line) - err_line = fs_str("{0}".format(stderr_line)) - if verbose and err_line is not None: - if spinner: - spinner.hide_and_write(err_line, target=spinner.stderr) - else: - sys.stderr.write(err_line) - sys.stderr.flush() - if stdout_line is not None: - output.append(stdout_line) - display_line = fs_str("{0}".format(stdout_line)) - if len(stdout_line) > display_limit: - display_line = "{0}...".format(stdout_line[:display_limit]) - if verbose and display_line is not None: - if spinner: - target = spinner.stdout if write_to_stdout else spinner.stderr - spinner.hide_and_write(display_line, target=target) - else: - target = sys.stdout if write_to_stdout else sys.stderr - target.write(display_line) - target.flush() - if spinner: - spinner.text = to_native_string("{0} {1}".format(spinner_orig_text, display_line)) - continue + spinner_orig_text = "" + if spinner and getattr(spinner, "text", None) is not None: + spinner_orig_text = spinner.text + if not spinner_orig_text and start_text is not None: + spinner_orig_text = start_text + stream_results = get_stream_results( + c, + verbose=verbose, + maxlen=display_limit, + spinner=spinner, + stdout_allowed=write_to_stdout, + ) try: c.wait() finally: @@ -237,6 +247,8 @@ def _create_subprocess( spinner.ok(to_native_string("✔ Complete")) else: spinner.ok(to_native_string("Complete")) + output = stream_results["stdout"] + err = stream_results["stderr"] c.out = "\n".join(output) if output else "" c.err = "\n".join(err) if err else "" else: @@ -261,7 +273,7 @@ def run( spinner_name=None, combine_stderr=True, display_limit=200, - write_to_stdout=True + write_to_stdout=True, ): """Use `subprocess.Popen` to get the output of a command and decode it. @@ -303,8 +315,12 @@ def run( if block or not return_object: combine_stderr = False start_text = "" - with spinner(spinner_name=spinner_name, start_text=start_text, nospin=nospin, - write_to_stdout=write_to_stdout) as sp: + with spinner( + spinner_name=spinner_name, + start_text=start_text, + nospin=nospin, + write_to_stdout=write_to_stdout, + ) as sp: return _create_subprocess( cmd, env=_env, @@ -315,7 +331,7 @@ def run( spinner=sp, combine_stderr=combine_stderr, start_text=start_text, - write_to_stdout=True + write_to_stdout=True, ) @@ -331,8 +347,9 @@ def load_path(python): """ python = Path(python).as_posix() - out, err = run([python, "-c", "import json, sys; print(json.dumps(sys.path))"], - nospin=True) + out, err = run( + [python, "-c", "import json, sys; print(json.dumps(sys.path))"], nospin=True + ) if out: return json.loads(out) else: @@ -445,7 +462,7 @@ def to_text(string, encoding="utf-8", errors=None): string = six.text_type(bytes(string), encoding, errors) else: string = string.decode(encoding, errors) - except UnicodeDecodeError as e: + except UnicodeDecodeError: string = " ".join(to_text(arg, encoding, errors) for arg in string) return string @@ -528,8 +545,8 @@ def get_output_encoding(source_encoding): """ if source_encoding is not None: - if get_canonical_encoding_name(source_encoding) == 'ascii': - return 'utf-8' + if get_canonical_encoding_name(source_encoding) == "ascii": + return "utf-8" return get_canonical_encoding_name(source_encoding) return get_canonical_encoding_name(PREFERRED_ENCODING) @@ -542,7 +559,7 @@ def _encode(output, encoding=None, errors=None, translation_map=None): except (UnicodeDecodeError, UnicodeEncodeError): if translation_map is not None: if six.PY2: - output = unicode.translate( + output = unicode.translate( # noqa: F821 to_text(output, encoding=encoding, errors=errors), translation_map ) else: @@ -573,8 +590,9 @@ def decode_for_output(output, target_stream=None, translation_map=None): try: output = _encode(output, encoding=encoding, translation_map=translation_map) except (UnicodeDecodeError, UnicodeEncodeError): - output = _encode(output, encoding=encoding, errors="replace", - translation_map=translation_map) + output = _encode( + output, encoding=encoding, errors="replace", translation_map=translation_map + ) return to_text(output, encoding=encoding, errors="replace") @@ -589,6 +607,7 @@ def get_canonical_encoding_name(name): """ import codecs + try: codec = codecs.lookup(name) except LookupError: @@ -629,8 +648,9 @@ class StreamWrapper(io.TextIOWrapper): # borrowed from click's implementation of stream wrappers, see # https://github.com/pallets/click/blob/6cafd32/click/_compat.py#L64 if six.PY2: + def write(self, x): - if isinstance(x, (str, buffer, bytearray)): + if isinstance(x, (str, buffer, bytearray)): # noqa: F821 try: self.flush() except Exception: diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index d3c8befe..d551aac7 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import absolute_import, print_function, unicode_literals import atexit import errno @@ -11,23 +11,21 @@ import stat import warnings import six - from six.moves import urllib_parse from six.moves.urllib import request as urllib_request from .backports.tempfile import _TemporaryFileWrapper from .compat import ( - _NamedTemporaryFile, Path, ResourceWarning, TemporaryDirectory, _fs_encoding, + _NamedTemporaryFile, finalize, fs_decode, - fs_encode + fs_encode, ) - __all__ = [ "check_for_unc_path", "get_converted_relative_path", @@ -51,7 +49,11 @@ __all__ = [ if os.name == "nt": - warnings.filterwarnings("ignore", category=DeprecationWarning, message="The Windows bytes API has been deprecated.*") + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + message="The Windows bytes API has been deprecated.*", + ) def unicode_path(path): @@ -93,9 +95,11 @@ def normalize_path(path): :rtype: str """ - return os.path.normpath(os.path.normcase( - os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) - )) + return os.path.normpath( + os.path.normcase( + os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) + ) + ) def is_in_path(path, parent): @@ -233,7 +237,7 @@ def mkdir_p(newdir, mode=0o777): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): raise OSError( - "A file with the same name as the desired dir, '{0}', already exists.".format( + "A file with the same name as the desired dir, '{0}', already exists.".format( fs_decode(newdir) ) ) @@ -307,7 +311,7 @@ def set_write_bit(fn): except AttributeError: pass for root, dirs, files in os.walk(fn, topdown=False): - for dir_ in [os.path.join(root,d) for d in dirs]: + for dir_ in [os.path.join(root, d) for d in dirs]: set_write_bit(dir_) for file_ in [os.path.join(root, f) for f in files]: set_write_bit(file_) @@ -332,9 +336,7 @@ def rmtree(directory, ignore_errors=False, onerror=None): if onerror is None: onerror = handle_remove_readonly try: - shutil.rmtree( - directory, ignore_errors=ignore_errors, onerror=onerror - ) + shutil.rmtree(directory, ignore_errors=ignore_errors, onerror=onerror) except (IOError, OSError, FileNotFoundError) as exc: # Ignore removal failures where the file doesn't exist if exc.errno != errno.ENOENT: @@ -355,14 +357,10 @@ def handle_remove_readonly(func, path, exc): :func:`set_write_bit` on the target path and try again. """ # Check for read-only attribute - from .compat import ( - ResourceWarning, FileNotFoundError, PermissionError - ) + from .compat import ResourceWarning, FileNotFoundError, PermissionError PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) - default_warning_message = ( - "Unable to remove file due to permissions restriction: {!r}" - ) + default_warning_message = "Unable to remove file due to permissions restriction: {!r}" # split the initial exception out into its type, exception, and traceback exc_type, exc_exception, exc_tb = exc if is_readonly_path(path): @@ -488,8 +486,8 @@ def get_converted_relative_path(path, relative_to=None): raise ValueError("The path argument does not currently accept UNC paths") relpath_s = to_text(posixpath.normpath(path.as_posix())) - if not (relpath_s == u"." or relpath_s.startswith(u"./")): - relpath_s = posixpath.join(u".", relpath_s) + if not (relpath_s == "." or relpath_s.startswith("./")): + relpath_s = posixpath.join(".", relpath_s) return relpath_s diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index 81ae3014..40dfff96 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -7,15 +7,15 @@ import signal import sys import threading import time +from io import StringIO import colorama import six from .compat import to_native_string from .cursor import hide_cursor, show_cursor -from .termcolors import COLOR_MAP, COLORS, colored, DISABLE_COLORS from .misc import decode_for_output -from io import StringIO +from .termcolors import COLOR_MAP, COLORS, DISABLE_COLORS, colored try: import yaspin @@ -26,10 +26,12 @@ except ImportError: else: import yaspin.spinners import yaspin.core + Spinners = yaspin.spinners.Spinners SpinBase = yaspin.core.Yaspin if os.name == "nt": + def handler(signum, frame, spinner): """Signal handler, used to gracefully shut down the ``spinner`` instance when specified signal is received by the process running the ``spinner``. @@ -41,7 +43,9 @@ if os.name == "nt": spinner.stop() sys.exit(0) + else: + def handler(signum, frame, spinner): """Signal handler, used to gracefully shut down the ``spinner`` instance when specified signal is received by the process running the ``spinner``. @@ -53,12 +57,10 @@ else: spinner.stop() sys.exit(0) + CLEAR_LINE = chr(27) + "[K" -TRANSLATION_MAP = { - 10004: u"OK", - 10008: u"x", -} +TRANSLATION_MAP = {10004: u"OK", 10008: u"x"} decode_output = functools.partial(decode_for_output, translation_map=TRANSLATION_MAP) @@ -84,6 +86,7 @@ class DummySpinner(object): def __exit__(self, exc_type, exc_val, tb): if exc_type: import traceback + formatted_tb = traceback.format_exception(exc_type, exc_val, tb) self.write_err("".join(formatted_tb)) self._close_output_buffer() @@ -197,10 +200,7 @@ class VistirSpinner(SpinBase): colorama.init() sigmap = {} if handler: - sigmap.update({ - signal.SIGINT: handler, - signal.SIGTERM: handler - }) + sigmap.update({signal.SIGINT: handler, signal.SIGTERM: handler}) handler_map = kwargs.pop("handler_map", {}) if os.name == "nt": sigmap[signal.SIGBREAK] = handler @@ -332,10 +332,7 @@ class VistirSpinner(SpinBase): def _compose_color_func(self): fn = functools.partial( - colored, - color=self._color, - on_color=self._on_color, - attrs=list(self._attrs), + colored, color=self._color, on_color=self._on_color, attrs=list(self._attrs) ) return fn diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py index d72e8ebe..6aecec88 100644 --- a/pipenv/vendor/vistir/termcolors.py +++ b/pipenv/vendor/vistir/termcolors.py @@ -1,62 +1,55 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals -import colorama + import os + +import colorama + from .compat import to_native_string - -DISABLE_COLORS = os.getenv("CI", False) or os.getenv("ANSI_COLORS_DISABLED", - os.getenv("VISTIR_DISABLE_COLORS", False) +DISABLE_COLORS = os.getenv("CI", False) or os.getenv( + "ANSI_COLORS_DISABLED", os.getenv("VISTIR_DISABLE_COLORS", False) ) ATTRIBUTES = dict( - list(zip([ - 'bold', - 'dark', - '', - 'underline', - 'blink', - '', - 'reverse', - 'concealed' - ], - list(range(1, 9)) - )) + list( + zip( + ["bold", "dark", "", "underline", "blink", "", "reverse", "concealed"], + list(range(1, 9)), ) -del ATTRIBUTES[''] + ) +) +del ATTRIBUTES[""] HIGHLIGHTS = dict( - list(zip([ - 'on_grey', - 'on_red', - 'on_green', - 'on_yellow', - 'on_blue', - 'on_magenta', - 'on_cyan', - 'on_white' + list( + zip( + [ + "on_grey", + "on_red", + "on_green", + "on_yellow", + "on_blue", + "on_magenta", + "on_cyan", + "on_white", ], - list(range(40, 48)) - )) + list(range(40, 48)), ) + ) +) COLORS = dict( - list(zip([ - 'grey', - 'red', - 'green', - 'yellow', - 'blue', - 'magenta', - 'cyan', - 'white', - ], - list(range(30, 38)) - )) + list( + zip( + ["grey", "red", "green", "yellow", "blue", "magenta", "cyan", "white"], + list(range(30, 38)), ) + ) +) COLOR_MAP = { @@ -106,11 +99,11 @@ def colored(text, color=None, on_color=None, attrs=None): colored('Hello, World!', 'red', 'on_grey', ['blue', 'blink']) colored('Hello, World!', 'green') """ - if os.getenv('ANSI_COLORS_DISABLED') is None: + if os.getenv("ANSI_COLORS_DISABLED") is None: style = "NORMAL" - if 'bold' in attrs: + if "bold" in attrs: style = "BRIGHT" - attrs.remove('bold') + attrs.remove("bold") if color is not None: color = color.upper() text = to_native_string("%s%s%s%s%s") % ( @@ -131,10 +124,7 @@ def colored(text, color=None, on_color=None, attrs=None): ) if attrs is not None: - fmt_str = to_native_string("%s[%%dm%%s%s[9m") % ( - chr(27), - chr(27) - ) + fmt_str = to_native_string("%s[%%dm%%s%s[9m") % (chr(27), chr(27)) for attr in attrs: text = fmt_str % (ATTRIBUTES[attr], text) diff --git a/pipenv/vendor/yaspin/__version__.py b/pipenv/vendor/yaspin/__version__.py index 9e78220f..f075dd36 100644 --- a/pipenv/vendor/yaspin/__version__.py +++ b/pipenv/vendor/yaspin/__version__.py @@ -1 +1 @@ -__version__ = "0.14.0" +__version__ = "0.14.1" diff --git a/pipenv/vendor/yaspin/api.py b/pipenv/vendor/yaspin/api.py index 156630db..f59ce002 100644 --- a/pipenv/vendor/yaspin/api.py +++ b/pipenv/vendor/yaspin/api.py @@ -84,5 +84,7 @@ def kbi_safe_yaspin(*args, **kwargs): return Yaspin(*args, **kwargs) -_kbi_safe_doc = yaspin.__doc__.replace("yaspin", "kbi_safe_yaspin") -kbi_safe_yaspin.__doc__ = _kbi_safe_doc +# Handle PYTHONOPTIMIZE=2 case, when docstrings are set to None. +if yaspin.__doc__: + _kbi_safe_doc = yaspin.__doc__.replace("yaspin", "kbi_safe_yaspin") + kbi_safe_yaspin.__doc__ = _kbi_safe_doc From c53e29b531b73119b721454cabf1110eb0bf45f4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 4 Mar 2019 20:44:28 -0500 Subject: [PATCH 18/21] Fix vendoring script in azure pipeline Signed-off-by: Dan Ryan --- .azure-pipelines/jobs/run-vendor-scripts.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/jobs/run-vendor-scripts.yml b/.azure-pipelines/jobs/run-vendor-scripts.yml index 5af6f866..a419d941 100644 --- a/.azure-pipelines/jobs/run-vendor-scripts.yml +++ b/.azure-pipelines/jobs/run-vendor-scripts.yml @@ -10,9 +10,8 @@ jobs: maxParallel: 4 matrix: ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - # TODO remove once vs2017-win2016 has Python 3.7 Python37: - python.version: '>= 3.7.0-b2' + python.version: '>= 3.7.2' python.architecture: x64 ${{ if ne(parameters.vmImage, 'vs2017-win2016' )}}: Python37: @@ -33,7 +32,7 @@ jobs: pip install certifi export GIT_SSL_CAINFO=$(python -m certifi) export LANG=C.UTF-8 - python -m pip install --upgrade invoke requests parver bs4 vistir towncrier + python -m pip install --upgrade invoke requests parver bs4 vistir towncrier pip setuptools wheel --upgrade-strategy=eager python -m invoke vendoring.update - template: ./run-manifest-check.yml From 01444dae287d655c7fb239fa24ac297ee4e610a9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 4 Mar 2019 20:47:07 -0500 Subject: [PATCH 19/21] Update news files Signed-off-by: Dan Ryan --- news/3298.bugfix.rst | 4 ++-- news/3298.vendor.rst | 4 ++-- news/3427.bugfix.rst | 2 +- news/3446.trivial.rst | 2 +- news/3449.bugfix.rst | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/news/3298.bugfix.rst b/news/3298.bugfix.rst index 1e27283a..aa378723 100644 --- a/news/3298.bugfix.rst +++ b/news/3298.bugfix.rst @@ -1,3 +1,3 @@ -- Fixed a bug which caused failures in warning reporting when running pipenv inside a virtualenv under some circumstances. +Fixed a bug which caused failures in warning reporting when running pipenv inside a virtualenv under some circumstances. -Fixed a bug with package discovery when running ``pipenv clean``. +- Fixed a bug with package discovery when running ``pipenv clean``. diff --git a/news/3298.vendor.rst b/news/3298.vendor.rst index 79fb60b9..40f9b510 100644 --- a/news/3298.vendor.rst +++ b/news/3298.vendor.rst @@ -1,4 +1,4 @@ -- Updated vendored dependencies: +Updated vendored dependencies: - **attrs**: ``18.2.0`` => ``19.1.0`` - **certifi**: ``2018.10.15`` => ``2018.11.29`` @@ -25,4 +25,4 @@ - **vistir**: ``0.3.0`` => ``0.3.1`` - **yaspin**: ``0.14.0`` => ``0.14.1`` -Removed vendored dependency **cursor**. +- Removed vendored dependency **cursor**. diff --git a/news/3427.bugfix.rst b/news/3427.bugfix.rst index 42f659f0..76aeb489 100644 --- a/news/3427.bugfix.rst +++ b/news/3427.bugfix.rst @@ -1 +1 @@ -Fix a bug that ``ValidationError`` is thrown when some fields are missing in source section. +Fixed a bug that ``ValidationError`` is thrown when some fields are missing in source section. diff --git a/news/3446.trivial.rst b/news/3446.trivial.rst index 1eb98bbb..c3f6a000 100644 --- a/news/3446.trivial.rst +++ b/news/3446.trivial.rst @@ -1 +1 @@ -Fix the wrong order of old and new hashes in message. +Fixed the wrong order of old and new hashes in message. diff --git a/news/3449.bugfix.rst b/news/3449.bugfix.rst index 72e8d228..4ed07046 100644 --- a/news/3449.bugfix.rst +++ b/news/3449.bugfix.rst @@ -1 +1 @@ -Update the index names in lock file when source name in Pipfile is changed. +Updated the index names in lock file when source name in Pipfile is changed. From 439a2581ad6200c051f6e319d32442f5337d588c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 4 Mar 2019 20:47:48 -0500 Subject: [PATCH 20/21] One more news file update Signed-off-by: Dan Ryan --- news/3298.feature.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/news/3298.feature.rst b/news/3298.feature.rst index 8aaf4e23..65a49424 100644 --- a/news/3298.feature.rst +++ b/news/3298.feature.rst @@ -1,5 +1,5 @@ -- Added full support for resolution of all dependency types including direct URLs, zip archives, tarballs, etc. +Added full support for resolution of all dependency types including direct URLs, zip archives, tarballs, etc. -Improved error handling and formatting. +- Improved error handling and formatting. -Introduced improved cross platform stream wrappers for better ``stdout`` and ``stderr`` consistency. +- Introduced improved cross platform stream wrappers for better ``stdout`` and ``stderr`` consistency. From b395aca762ea4882bfc961b2e1e26cc5acadd8f2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 5 Mar 2019 02:14:48 -0500 Subject: [PATCH 21/21] Add missing news entries Signed-off-by: Dan Ryan --- news/2722.bugfix.rst | 1 + news/3053.bugfix.rst | 1 + news/3071.bugfix.rst | 1 + news/3328.feature.rst | 1 + news/3368.feature.rst | 1 + news/3404.bugfix.rst | 1 + 6 files changed, 6 insertions(+) create mode 100644 news/2722.bugfix.rst create mode 100644 news/3053.bugfix.rst create mode 100644 news/3071.bugfix.rst create mode 100644 news/3328.feature.rst create mode 100644 news/3368.feature.rst create mode 100644 news/3404.bugfix.rst diff --git a/news/2722.bugfix.rst b/news/2722.bugfix.rst new file mode 100644 index 00000000..8c26df8d --- /dev/null +++ b/news/2722.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which caused editable package resolution to sometimes fail with an unhelpful setuptools-related error message. diff --git a/news/3053.bugfix.rst b/news/3053.bugfix.rst new file mode 100644 index 00000000..21134f59 --- /dev/null +++ b/news/3053.bugfix.rst @@ -0,0 +1 @@ +Dependency resolution now writes hashes for local and remote files to the lockfile. diff --git a/news/3071.bugfix.rst b/news/3071.bugfix.rst new file mode 100644 index 00000000..dd4145ea --- /dev/null +++ b/news/3071.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which prevented ``pipenv graph`` from correctly showing all dependencies when running from within ``pipenv shell``. diff --git a/news/3328.feature.rst b/news/3328.feature.rst new file mode 100644 index 00000000..7e92d39f --- /dev/null +++ b/news/3328.feature.rst @@ -0,0 +1 @@ +Pipenv will now successfully recursively lock VCS sub-dependencies. diff --git a/news/3368.feature.rst b/news/3368.feature.rst new file mode 100644 index 00000000..a998fce1 --- /dev/null +++ b/news/3368.feature.rst @@ -0,0 +1 @@ +Pipenv will now discover and resolve the intrinsic dependencies of **all** VCS dependencies, whether they are editable or not, to prevent resolution conflicts. diff --git a/news/3404.bugfix.rst b/news/3404.bugfix.rst new file mode 100644 index 00000000..fa678d6a --- /dev/null +++ b/news/3404.bugfix.rst @@ -0,0 +1 @@ +Fixed a keyerror which could occur when locking VCS dependencies in some cases.