From 01845e5f511cdc7f7d06cab5bc69bbab8b33c352 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 14 Feb 2019 20:38:16 -0500 Subject: [PATCH] Update requirementslib for PEP517 builder Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/__init__.py | 2 +- .../vendor/requirementslib/models/pipfile.py | 42 +- .../requirementslib/models/requirements.py | 641 ++++++++---------- .../requirementslib/models/setup_info.py | 433 ++++++++---- pipenv/vendor/requirementslib/models/utils.py | 132 ++-- pipenv/vendor/requirementslib/utils.py | 57 +- 6 files changed, 687 insertions(+), 620 deletions(-) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index a8b4a19c..c7156971 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,6 +1,6 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function -__version__ = '1.4.0' +__version__ = '1.4.1.dev0' import logging import warnings diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index f04f5ba1..021b5d53 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -22,12 +22,12 @@ from .utils import optional_instance_of, get_url_name from ..environment import MYPY_RUNNING if MYPY_RUNNING: - from typing import Union, Any, Dict, Iterable, Sequence, Mapping, List, NoReturn - package_type = Dict[str, Dict[str, Union[List[str], str]]] - source_type = Dict[str, Union[str, bool]] + from typing import Union, Any, Dict, Iterable, Sequence, Mapping, List, NoReturn, Text + package_type = Dict[Text, Dict[Text, Union[List[Text], Text]]] + source_type = Dict[Text, Union[Text, bool]] sources_type = Iterable[source_type] - meta_type = Dict[str, Union[int, Dict[str, str], sources_type]] - lockfile_type = Dict[str, Union[package_type, meta_type]] + meta_type = Dict[Text, Union[int, Dict[Text, Text], sources_type]] + lockfile_type = Dict[Text, Union[package_type, meta_type]] # Let's start by patching plette to make sure we can validate data without being broken @@ -45,7 +45,7 @@ def patch_plette(): global VALIDATORS def validate(cls, data): - # type: (Any, Dict[str, Any]) -> None + # type: (Any, Dict[Text, Any]) -> None if not cerberus: # Skip validation if Cerberus is not available. return schema = cls.__SCHEMA__ @@ -98,7 +98,7 @@ def reorder_source_keys(data): class PipfileLoader(plette.pipfiles.Pipfile): @classmethod def validate(cls, data): - # type: (Dict[str, Any]) -> None + # type: (Dict[Text, Any]) -> None for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items(): if key not in data or key == "source": continue @@ -121,7 +121,7 @@ class PipfileLoader(plette.pipfiles.Pipfile): @classmethod def load(cls, f, encoding=None): - # type: (Any, str) -> PipfileLoader + # type: (Any, Text) -> PipfileLoader content = f.read() if encoding is not None: content = content.decode(encoding) @@ -145,7 +145,7 @@ class PipfileLoader(plette.pipfiles.Pipfile): return instance def __getattribute__(self, key): - # type: (str) -> Any + # type: (Text) -> Any if key == "source": return self._data[key] return super(PipfileLoader, self).__getattribute__(key) @@ -182,8 +182,8 @@ class Pipfile(object): return self._pipfile def get_deps(self, dev=False, only=True): - # type: (bool, bool) -> Dict[str, Dict[str, Union[List[str], str]]] - deps = {} # type: Dict[str, Dict[str, Union[List[str], str]]] + # type: (bool, bool) -> Dict[Text, Dict[Text, Union[List[Text], Text]]] + deps = {} # type: Dict[Text, Dict[Text, Union[List[Text], Text]]] if dev: deps.update(self.pipfile._data["dev-packages"]) if only: @@ -191,11 +191,11 @@ class Pipfile(object): return merge_items([deps, self.pipfile._data["packages"]]) def get(self, k): - # type: (str) -> Any + # type: (Text) -> Any return self.__getitem__(k) def __contains__(self, k): - # type: (str) -> bool + # type: (Text) -> bool check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) if check_pipfile: return True @@ -247,10 +247,10 @@ class Pipfile(object): @classmethod def read_projectfile(cls, path): - # type: (str) -> ProjectFile + # type: (Text) -> ProjectFile """Read the specified project file and provide an interface for writing/updating. - :param str path: Path to the target file. + :param Text path: Path to the target file. :return: A project file with the model and location for interaction :rtype: :class:`~requirementslib.models.project.ProjectFile` """ @@ -263,10 +263,10 @@ class Pipfile(object): @classmethod def load_projectfile(cls, path, create=False): - # type: (str, bool) -> ProjectFile + # type: (Text, bool) -> ProjectFile """Given a path, load or create the necessary pipfile. - :param str path: Path to the project root or pipfile + :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` @@ -288,10 +288,10 @@ class Pipfile(object): @classmethod def load(cls, path, create=False): - # type: (str, bool) -> Pipfile + # type: (Text, bool) -> Pipfile """Given a path, load or create the necessary pipfile. - :param str path: Path to the project root or pipfile + :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` @@ -347,10 +347,10 @@ class Pipfile(object): @property def build_requires(self): - # type: () -> List[str] + # type: () -> List[Text] return self.build_system.get("requires", []) @property def build_backend(self): - # type: () -> str + # type: () -> Text return self.build_system.get("build-backend", None) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index be126e28..b25a6e1c 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -28,7 +28,7 @@ from packaging.specifiers import Specifier, SpecifierSet, LegacySpecifier, Inval from packaging.utils import canonicalize_name from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote -from vistir.compat import Path +from vistir.compat import Path, Iterable, FileNotFoundError from vistir.contextmanagers import temp_path from vistir.misc import dedup from vistir.path import ( @@ -81,11 +81,12 @@ from .utils import ( from ..environment import MYPY_RUNNING if MYPY_RUNNING: - from typing import Optional, TypeVar, List, Dict, Union, Any, Tuple, Generator, Set + from typing import Optional, TypeVar, List, Dict, Union, Any, Tuple, Generator, Set, Text from pip_shims.shims import Link, InstallRequirement RequirementType = TypeVar('RequirementType', covariant=True, bound=PackagingRequirement) from six.moves.urllib.parse import SplitResult from .vcs import VCSRepository + NON_STRING_ITERABLE = Union[List, Set, Tuple] SPECIFIERS_BY_LENGTH = sorted(list(Specifier._operators.keys()), key=len, reverse=True) @@ -96,41 +97,41 @@ run = partial(vistir.misc.run, combine_stderr=False, return_object=True, nospin= class Line(object): def __init__(self, line, extras=None): - # type: (str) -> None + # type: (Text, Optional[NON_STRING_ITERABLE]) -> None self.editable = False # type: bool if line.startswith("-e "): line = line[len("-e "):] self.editable = True - self.extras = () # type: Tuple[str] + self.extras = () # type: Tuple[Text] if extras is not None: self.extras = tuple(sorted(set(extras))) - self.line = line # type: str - self.hashes = [] # type: List[str] - self.markers = None # type: Optional[str] - self.vcs = None # type: Optional[str] - self.path = None # type: Optional[str] - self.relpath = None # type: Optional[str] - self.uri = None # type: Optional[str] + self.line = line # type: Text + self.hashes = [] # type: List[Text] + self.markers = None # type: Optional[Text] + self.vcs = None # type: Optional[Text] + self.path = None # type: Optional[Text] + self.relpath = None # type: Optional[Text] + self.uri = None # type: Optional[Text] self._link = None # type: Optional[Link] self.is_local = False # type: bool - self._name = None # type: Optional[str] - self._specifier = None # type: Optional[str] + self._name = None # type: Optional[Text] + self._specifier = None # type: Optional[Text] self.parsed_marker = None # type: Optional[Marker] - self.preferred_scheme = None # type: Optional[str] + self.preferred_scheme = None # type: Optional[Text] self._requirement = None # type: Optional[PackagingRequirement] self.is_direct_url = False # type: bool self._parsed_url = None # type: Optional[urllib_parse.ParseResult] - self._setup_cfg = None # type: Optional[str] - self._setup_py = None # type: Optional[str] - self._pyproject_toml = None # type: Optional[str] - self._pyproject_requires = None # type: Optional[List[str]] - self._pyproject_backend = None # type: Optional[str] - self._wheel_kwargs = None # type: Dict[str, str] + self._setup_cfg = None # type: Optional[Text] + self._setup_py = None # type: Optional[Text] + self._pyproject_toml = None # type: Optional[Text] + self._pyproject_requires = None # type: Optional[List[Text]] + self._pyproject_backend = None # type: Optional[Text] + self._wheel_kwargs = None # type: Dict[Text, Text] self._vcsrepo = None # type: Optional[VCSRepository] self._setup_info = None # type: Optional[SetupInfo] - self._ref = None # type: Optional[str] + self._ref = None # type: Optional[Text] self._ireq = None # type: Optional[InstallRequirement] - self._src_root = None # type: Optional[str] + self._src_root = None # type: Optional[Text] self.dist = None # type: Any super(Line, self).__init__() self.parse() @@ -156,12 +157,12 @@ class Line(object): @classmethod def split_hashes(cls, line): - # type: (str) -> Tuple[str, List[str]] + # type: (Text) -> Tuple[Text, List[Text]] if "--hash" not in line: return line, [] split_line = line.split() - line_parts = [] # type: List[str] - hashes = [] # type: List[str] + line_parts = [] # type: List[Text] + hashes = [] # type: List[Text] for part in split_line: if part.startswith("--hash"): param, _, value = part.partition("=") @@ -173,7 +174,7 @@ class Line(object): @property def line_with_prefix(self): - # type: () -> str + # type: () -> Text line = self.line extras_str = extras_to_string(self.extras) if self.is_direct_url: @@ -193,7 +194,7 @@ class Line(object): @property def line_for_ireq(self): - # type: () -> str + # type: () -> Text line = "" if self.is_file or self.is_url and not self.is_vcs: scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri" @@ -232,7 +233,7 @@ class Line(object): @property def base_path(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if not self.link and not self.path: self.parse_link() if not self.path: @@ -248,28 +249,28 @@ class Line(object): @property def setup_py(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self._setup_py is None: self.populate_setup_paths() return self._setup_py @property def setup_cfg(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self._setup_cfg is None: self.populate_setup_paths() return self._setup_cfg @property def pyproject_toml(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self._pyproject_toml is None: self.populate_setup_paths() return self._pyproject_toml @property def specifier(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] options = [self._specifier] for req in (self.ireq, self.requirement): if req is not None and getattr(req, "specifier", None): @@ -277,13 +278,21 @@ class Line(object): specifier = next(iter(spec for spec in options if spec is not None), None) if specifier is not None: specifier = specs_to_string(specifier) - elif specifier is None and not self.is_named and self.setup_info is not None: - if self.setup_info.version: - specifier = "=={0}".format(self.setup_info.version) + elif specifier is None and not self.is_named and self._setup_info is not None: + if self._setup_info.version: + specifier = "=={0}".format(self._setup_info.version) if specifier: self._specifier = specifier return self._specifier + @specifier.setter + def specifier(self, spec): + # type: (str) -> None + if not spec.startswith("=="): + spec = "=={0}".format(spec) + self._specifier = spec + self.specifiers = SpecifierSet(spec) + @property def specifiers(self): # type: () -> Optional[SpecifierSet] @@ -308,7 +317,7 @@ class Line(object): @specifiers.setter def specifiers(self, specifiers): - # type: (Union[str, SpecifierSet]) -> None + # type: (Union[Text, SpecifierSet]) -> None if type(specifiers) is not SpecifierSet: if type(specifiers) in six.string_types: specifiers = SpecifierSet(specifiers) @@ -324,7 +333,7 @@ class Line(object): @classmethod def get_requirement_specs(cls, specifierset): - # type: (SpecifierSet) -> List[Tuple[str, str]] + # type: (SpecifierSet) -> List[Tuple[Text, Text]] specs = [] spec = next(iter(specifierset._specs), None) if spec: @@ -354,14 +363,14 @@ class Line(object): base_path = self.base_path if base_path is None: return - setup_paths = get_setup_paths(self.base_path, subdirectory=self.subdirectory) # type: Dict[str, Optional[str]] + setup_paths = get_setup_paths(self.base_path, subdirectory=self.subdirectory) # type: Dict[Text, Optional[Text]] self._setup_py = setup_paths.get("setup_py") self._setup_cfg = setup_paths.get("setup_cfg") self._pyproject_toml = setup_paths.get("pyproject_toml") @property def pyproject_requires(self): - # type: () -> Optional[List[str]] + # type: () -> Optional[List[Text]] if self._pyproject_requires is None and self.pyproject_toml is not None: pyproject_requires, pyproject_backend = get_pyproject(self.path) self._pyproject_requires = pyproject_requires @@ -370,7 +379,7 @@ class Line(object): @property def pyproject_backend(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self._pyproject_requires is None and self.pyproject_toml is not None: pyproject_requires, pyproject_backend = get_pyproject(self.path) if not pyproject_backend and self.setup_cfg is not None: @@ -482,7 +491,7 @@ class Line(object): self.extras = tuple(sorted(extras)) def get_url(self): - # type: () -> str + # type: () -> Text """Sets ``self.name`` if given a **PEP-508** style URL""" line = self.line @@ -510,19 +519,20 @@ class Line(object): @property def name(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self._name is None: self.parse_name() if self._name is None and not self.is_named and not self.is_wheel: - setup_info = self.setup_info - self._name = setup_info.name + if self.setup_info: + self._name = self.setup_info.name return self._name @name.setter def name(self, name): - # type: (str) -> None + # type: (Text) -> None self._name = name - self._setup_info.name = name + if self._setup_info: + self._setup_info.name = name if self.requirement: self._requirement.name = name if self.ireq and self.ireq.req: @@ -530,7 +540,7 @@ class Line(object): @property def url(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self.uri is not None: url = add_ssh_scheme_to_git_uri(self.uri) else: @@ -555,7 +565,7 @@ class Line(object): @property def subdirectory(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self.link is not None: return self.link.subdirectory_fragment return "" @@ -633,7 +643,7 @@ class Line(object): @property def ref(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self._ref is None and self.relpath is not None: self.relpath, self._ref = split_ref_from_uri(self.relpath) return self._ref @@ -657,21 +667,39 @@ class Line(object): self._wheel_kwargs = _prepare_wheel_building_kwargs(self.ireq) return self._wheel_kwargs + def get_setup_info(self): + # type: () -> SetupInfo + setup_info = SetupInfo.from_ireq(self.ireq) + if not setup_info.name: + setup_info.get_info() + return setup_info + @property def setup_info(self): # type: () -> Optional[SetupInfo] if self._setup_info is None and not self.is_named and not self.is_wheel: - self._setup_info = SetupInfo.from_ireq(self.ireq) - if self._setup_info is not None: - self._setup_info.get_info() + if self._setup_info: + if not self._setup_info.name: + self._setup_info.get_info() + else: + # make two attempts at this before failing to allow for stale data + try: + self.setup_info = self.get_setup_info() + except FileNotFoundError: + try: + self.setup_info = self.get_setup_info() + except FileNotFoundError: + raise return self._setup_info @setup_info.setter def setup_info(self, setup_info): # type: (SetupInfo) -> None self._setup_info = setup_info - if self._parsed_line: - self._parsed_line._setup_info = setup_info + if setup_info.version: + self.specifier = setup_info.version + if setup_info.name and not self.name: + self.name = setup_info.name def _get_vcsrepo(self): # type: () -> Optional[VCSRepository] @@ -731,40 +759,6 @@ class Line(object): ireq = pip_shims.shims.install_req_from_line(self.line) if self.is_file or self.is_url: ireq.link = self.link - # elif (self.is_file or self.is_url) and not self.is_vcs: - # line = self.line - # if self.is_direct_url: - # line = self.link.url - # scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri" - # local_line = next(iter([ - # os.path.dirname(os.path.abspath(f)) for f in [ - # self.setup_py, self.setup_cfg, self.pyproject_toml - # ] if f is not None - # ]), None) - # line = local_line if local_line is not None else self.line - # if scheme == "path": - # if not line and self.base_path is not None: - # line = os.path.abspath(self.base_path) - # else: - # if self.link is not None: - # line = self.link.url_without_fragment - # else: - # if self.uri is not None: - # line = self.uri - # else: - # line = self.path - # if self.editable: - # ireq = pip_shims.shims.install_req_from_editable(self.link.url) - # ireq.link = self.link - # else: - # ireq = pip_shims.shims.install_req_from_line(line) - # else: - # if self.editable: - # ireq = pip_shims.shims.install_req_from_editable(self.link.url) - # ireq.link = self.link - # else: - # ireq = pip_shims.shims.install_req_from_line(self.link.url) - # ireq.link = self.link if self.extras and not ireq.extras: ireq.extras = set(self.extras) if self.parsed_marker is not None and not ireq.markers: @@ -782,7 +776,7 @@ class Line(object): self._ireq.req = self.requirement def _parse_wheel(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if not self.is_wheel: pass from pip_shims.shims import Wheel @@ -793,7 +787,7 @@ class Line(object): return name def _parse_name_from_link(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self.link is None: return None @@ -804,7 +798,7 @@ class Line(object): return None def _parse_name_from_line(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if not self.is_named: pass @@ -813,7 +807,8 @@ class Line(object): except Exception: raise RequirementError("Failed parsing requirement from {0!r}".format(self.line)) name = self._requirement.name - self._specifier = specs_to_string(self._requirement.specifier) + if not self._specifier and self._requirement and self._requirement.specifier: + self._specifier = specs_to_string(self._requirement.specifier) if self._requirement.extras and not self.extras: self.extras = self._requirement.extras if not name: @@ -850,21 +845,6 @@ class Line(object): def _parse_requirement_from_vcs(self): # type: () -> Optional[PackagingRequirement] - # name = self._name if self._name else self.link.egg_fragment - # url = self.url if self.url else self.uri - # if self.is_direct_url: - # url = self.link.url - # if not name: - # raise ValueError( - # "pipenv requires an #egg fragment for version controlled " - # "dependencies. Please install remote dependency " - # "in the form {0}#egg=.".format(url) - # ) - # req = init_requirement(canonicalize_name(name)) # type: PackagingRequirement - # req.editable = self.editable - # if not getattr(req, "url") and self.link: - # req.url = url - # req.line = self.link.url if ( self.uri != unquote(self.url) and "git+ssh://" in self.url @@ -887,12 +867,6 @@ class Line(object): self._requirement.revision = self._vcsrepo.get_commit_hash() else: self._requirement.revision = self.ref - # if self.extras: - # req.extras = self.extras - # req.vcs = self.vcs - # if self.path and self.link and self.link.scheme.startswith("file"): - # req.local_file = True - # req.path = self.path return self._requirement def parse_requirement(self): @@ -900,8 +874,8 @@ class Line(object): if self._name is None: self.parse_name() if not self._name and not self.is_vcs and not self.is_named: - setup_info = self.setup_info - self._name = setup_info.name + if self.setup_info and self.setup_info.name: + self._name = self.setup_info.name name, extras, url = self.requirement_info if name: self._requirement = init_requirement(name) # type: PackagingRequirement @@ -939,28 +913,6 @@ class Line(object): "dependencies. Please install remote dependency " "in the form {0}#egg=.".format(url) ) - # if self.is_named: - # self._requirement = init_requirement(self.line) - # elif self.is_vcs: - # self._requirement = self._parse_requirement_from_vcs() - # if self._name is None and ( - # self._requirement is not None and self._requirement.name is not None - # ): - # self.name = self._requirement.name - # if self._name is not None and self._requirement is None: - # self._requirement = init_requirement(self._name) - # if self._requirement: - # if self.parsed_marker is not None: - # self._requirement.marker = self.parsed_marker - # if self.is_url or self.is_file and (self.link or self.url) and not self.is_vcs: - # if self.uri: - # self._requirement.url = self.url - # elif self.link: - # self._requirement.url = unquote(self.link.url_without_fragment) - # else: - # self._requirement.url = self.uri - # if self.extras and not self._requirement.extras: - # self._requirement.extras = set(self.extras) def parse_link(self): # type: () -> None @@ -1007,13 +959,13 @@ class Line(object): @property def requirement_info(self): - # type: () -> Tuple(Optional[str], Tuple[Optional[str]], Optional[str]) + # type: () -> Tuple(Optional[Text], Tuple[Optional[Text]], Optional[Text]) """ Generates a 3-tuple of the requisite *name*, *extras* and *url* to generate a :class:`~packaging.requirements.Requirement` out of. :return: A Tuple containing an optional name, a Tuple of extras names, and an optional URL. - :rtype: Tuple[Optional[str], Tuple[Optional[str]], Optional[str]] + :rtype: Tuple[Optional[Text], Tuple[Optional[Text]], Optional[Text]] """ # Direct URLs can be converted to packaging requirements directly, but @@ -1094,10 +1046,10 @@ class Line(object): @attr.s(slots=True, hash=True) class NamedRequirement(object): - name = attr.ib() # type: str - version = attr.ib() # type: Optional[str] + name = attr.ib() # type: Text + version = attr.ib() # type: Optional[Text] req = attr.ib() # type: PackagingRequirement - extras = attr.ib(default=attr.Factory(list)) # type: Tuple[str] + extras = attr.ib(default=attr.Factory(list)) # type: Tuple[Text] editable = attr.ib(default=False) # type: bool _parsed_line = attr.ib(default=None) # type: Optional[Line] @@ -1118,9 +1070,9 @@ class NamedRequirement(object): @classmethod def from_line(cls, line, parsed_line=None): - # type: (str, Optional[Line]) -> NamedRequirement + # type: (Text, Optional[Line]) -> NamedRequirement req = init_requirement(line) - specifiers = None # type: Optional[str] + specifiers = None # type: Optional[Text] if req.specifier: specifiers = specs_to_string(req.specifier) req.line = line @@ -1138,7 +1090,7 @@ class NamedRequirement(object): "parsed_line": parsed_line, "extras": None } - extras = None # type: Optional[Tuple[str]] + extras = None # type: Optional[Tuple[Text]] if req.extras: extras = list(req.extras) creation_kwargs["extras"] = extras @@ -1146,13 +1098,13 @@ class NamedRequirement(object): @classmethod def from_pipfile(cls, name, pipfile): - # type: (str, Dict[str, Union[str, Optional[str], Optional[List[str]]]]) -> NamedRequirement - creation_args = {} # type: Dict[str, Union[Optional[str], Optional[List[str]]]] + # type: (Text, Dict[Text, Union[Text, Optional[Text], Optional[List[Text]]]]) -> NamedRequirement + creation_args = {} # type: Dict[Text, Union[Optional[Text], Optional[List[Text]]]] if hasattr(pipfile, "keys"): attr_fields = [field.name for field in attr.fields(cls)] creation_args = {k: v for k, v in pipfile.items() if k in attr_fields} creation_args["name"] = name - version = get_version(pipfile) # type: Optional[str] + version = get_version(pipfile) # type: Optional[Text] extras = creation_args.get("extras", None) creation_args["version"] = version req = init_requirement("{0}{1}".format(name, version)) @@ -1163,7 +1115,7 @@ class NamedRequirement(object): @property def line_part(self): - # type: () -> str + # type: () -> Text # FIXME: This should actually be canonicalized but for now we have to # simply lowercase it and replace underscores, since full canonicalization # also replaces dots and that doesn't actually work when querying the index @@ -1171,7 +1123,7 @@ class NamedRequirement(object): @property def pipfile_part(self): - # type: () -> Dict[str, Any] + # type: () -> Dict[Text, Any] pipfile_dict = attr.asdict(self, filter=filter_none).copy() # type: ignore if "version" not in pipfile_dict: pipfile_dict["version"] = "*" @@ -1192,36 +1144,36 @@ class FileRequirement(object): containing directories.""" #: Path to the relevant `setup.py` location - setup_path = attr.ib(default=None, cmp=True) # type: Optional[str] + setup_path = attr.ib(default=None, cmp=True) # type: Optional[Text] #: path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - path = attr.ib(default=None, cmp=True) # type: Optional[str] + path = attr.ib(default=None, cmp=True) # type: Optional[Text] #: Whether the package is editable editable = attr.ib(default=False, cmp=True) # type: bool #: Extras if applicable - extras = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Tuple[str] - _uri_scheme = attr.ib(default=None, cmp=True) # type: Optional[str] + extras = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Tuple[Text] + _uri_scheme = attr.ib(default=None, cmp=True) # type: Optional[Text] #: URI of the package - uri = attr.ib(cmp=True) # type: Optional[str] + uri = attr.ib(cmp=True) # type: Optional[Text] #: Link object representing the package to clone link = attr.ib(cmp=True) # type: Optional[Link] #: PyProject Requirements pyproject_requires = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Tuple #: PyProject Build System - pyproject_backend = attr.ib(default=None, cmp=True) # type: Optional[str] + pyproject_backend = attr.ib(default=None, cmp=True) # type: Optional[Text] #: PyProject Path - pyproject_path = attr.ib(default=None, cmp=True) # type: Optional[str] + pyproject_path = attr.ib(default=None, cmp=True) # type: Optional[Text] #: Setup metadata e.g. dependencies _setup_info = attr.ib(default=None, cmp=True) # type: Optional[SetupInfo] _has_hashed_name = attr.ib(default=False, cmp=True) # type: bool _parsed_line = attr.ib(default=None, cmp=False, hash=True) # type: Optional[Line] #: Package name - name = attr.ib(cmp=True) # type: Optional[str] + name = attr.ib(cmp=True) # type: Optional[Text] #: A :class:`~pkg_resources.Requirement` isntance req = attr.ib(cmp=True) # type: Optional[PackagingRequirement] @classmethod def get_link_from_line(cls, line): - # type: (str) -> LinkInfo + # type: (Text) -> LinkInfo """Parse link information from given requirement line. Return a 6-tuple: @@ -1255,16 +1207,16 @@ class FileRequirement(object): # Git allows `git@github.com...` lines that are not really URIs. # Add "ssh://" so we can parse correctly, and restore afterwards. - fixed_line = add_ssh_scheme_to_git_uri(line) # type: str + fixed_line = add_ssh_scheme_to_git_uri(line) # type: Text added_ssh_scheme = fixed_line != line # type: bool # We can assume a lot of things if this is a local filesystem path. if "://" not in fixed_line: p = Path(fixed_line).absolute() # type: Path - path = p.as_posix() # type: Optional[str] - uri = p.as_uri() # type: str + path = p.as_posix() # type: Optional[Text] + uri = p.as_uri() # type: Text link = create_link(uri) # type: Link - relpath = None # type: Optional[str] + relpath = None # type: Optional[Text] try: relpath = get_converted_relative_path(path) except ValueError: @@ -1277,13 +1229,13 @@ class FileRequirement(object): original_url = parsed_url._replace() # type: SplitResult # Split the VCS part out if needed. - original_scheme = parsed_url.scheme # type: str - vcs_type = None # type: Optional[str] + original_scheme = parsed_url.scheme # type: Text + vcs_type = None # type: Optional[Text] if "+" in original_scheme: - scheme = None # type: Optional[str] + scheme = None # type: Optional[Text] vcs_type, _, scheme = original_scheme.partition("+") parsed_url = parsed_url._replace(scheme=scheme) - prefer = "uri" # type: str + prefer = "uri" # type: Text else: vcs_type = None prefer = "file" @@ -1323,17 +1275,17 @@ class FileRequirement(object): @property def setup_py_dir(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self.setup_path: return os.path.dirname(os.path.abspath(self.setup_path)) return None @property def dependencies(self): - # type: () -> Tuple[Dict[str, PackagingRequirement], List[Union[str, PackagingRequirement]], List[str]] - build_deps = [] # type: List[Union[str, PackagingRequirement]] - setup_deps = [] # type: List[str] - deps = {} # type: Dict[str, PackagingRequirement] + # type: () -> Tuple[Dict[Text, PackagingRequirement], List[Union[Text, PackagingRequirement]], List[Text]] + build_deps = [] # type: List[Union[Text, PackagingRequirement]] + setup_deps = [] # type: List[Text] + deps = {} # type: Dict[Text, PackagingRequirement] if self.setup_info: setup_info = self.setup_info.as_dict() deps.update(setup_info.get("requires", {})) @@ -1346,6 +1298,7 @@ class FileRequirement(object): return deps, setup_deps, build_deps def __attrs_post_init__(self): + # type: () -> None if self.name is None and self.parsed_line: if self.parsed_line.setup_info: self._setup_info = self.parsed_line.setup_info @@ -1359,26 +1312,31 @@ class FileRequirement(object): @property def setup_info(self): + # type: () -> SetupInfo from .setup_info import SetupInfo if self._setup_info is None and self.parsed_line: if self.parsed_line.setup_info: + if not self._parsed_line.setup_info.name: + self._parsed_line._setup_info.get_info() self._setup_info = self.parsed_line.setup_info elif self.parsed_line.ireq and not self.parsed_line.is_wheel: self._setup_info = SetupInfo.from_ireq(self.parsed_line.ireq) else: if self.link and not self.link.is_wheel: self._setup_info = Line(self.line_part).setup_info + self._setup_info.get_info() return self._setup_info @setup_info.setter def setup_info(self, setup_info): + # type: (SetupInfo) -> None self._setup_info = setup_info if self._parsed_line: self._parsed_line._setup_info = setup_info @uri.default def get_uri(self): - # type: () -> str + # type: () -> Text if self.path and not self.uri: self._uri_scheme = "path" return pip_shims.shims.path_to_url(os.path.abspath(self.path)) @@ -1390,7 +1348,7 @@ class FileRequirement(object): @name.default def get_name(self): - # type: () -> str + # type: () -> Text loc = self.path or self.uri if loc and not self._uri_scheme: self._uri_scheme = "path" if self.path else "file" @@ -1462,7 +1420,7 @@ class FileRequirement(object): @link.default def get_link(self): - # type: () -> Link + # type: () -> pip_shims.shims.Link target = "{0}".format(self.uri) if hasattr(self, "name") and not self._has_hashed_name: target = "{0}#egg={1}".format(target, self.name) @@ -1471,7 +1429,7 @@ class FileRequirement(object): @req.default def get_requirement(self): - # type: () -> PackagingRequirement + # type: () -> RequirementType if self.name is None: if self._parsed_line is not None and self._parsed_line.name is not None: self.name = self._parsed_line.name @@ -1557,7 +1515,7 @@ class FileRequirement(object): @property def formatted_path(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self.path: path = self.path if not isinstance(path, Path): @@ -1568,16 +1526,16 @@ class FileRequirement(object): @classmethod def create( cls, - path=None, # type: Optional[str] - uri=None, # type: str + path=None, # type: Optional[Text] + uri=None, # type: Text editable=False, # type: bool - extras=None, # type: Optional[Tuple[str]] + extras=None, # type: Optional[Tuple[Text]] link=None, # type: Link vcs_type=None, # type: Optional[Any] - name=None, # type: Optional[str] + name=None, # type: Optional[Text] req=None, # type: Optional[Any] - line=None, # type: Optional[str] - uri_scheme=None, # type: str + line=None, # type: Optional[Text] + uri_scheme=None, # type: Text setup_path=None, # type: Optional[Any] relpath=None, # type: Optional[Any] parsed_line=None, # type: Optional[Line] @@ -1638,16 +1596,16 @@ class FileRequirement(object): creation_kwargs["vcs"] = vcs_type if name: creation_kwargs["name"] = name - _line = None - ireq = None - setup_info = None + _line = None # type: Optional[Text] + ireq = None # type: Optional[InstallRequirement] + setup_info = None # type: Optional[SetupInfo] if parsed_line: if parsed_line.name: name = parsed_line.name if parsed_line.setup_info: name = parsed_line.setup_info.as_dict().get("name", name) if not name or not parsed_line: - if link is not None and link.url is not None: + if link is not None and link.url_without_fragment is not None: _line = unquote(link.url_without_fragment) if name: _line = "{0}#egg={1}".format(_line, name) @@ -1676,16 +1634,16 @@ class FileRequirement(object): parsed_line = Line(_line) if ireq is None: ireq = parsed_line.ireq - if extras and not ireq.extras: + if extras and ireq is not None and not ireq.extras: ireq.extras = set(extras) if setup_info is None: setup_info = SetupInfo.from_ireq(ireq) setupinfo_dict = setup_info.as_dict() setup_name = setupinfo_dict.get("name", None) - if setup_name: + if setup_name is not None: name = setup_name build_requires = setupinfo_dict.get("build_requires", ()) - build_backend = setupinfo_dict.get("build_backend", ()) + build_backend = setupinfo_dict.get("build_backend", "") if not creation_kwargs.get("pyproject_requires") and build_requires: creation_kwargs["pyproject_requires"] = tuple(build_requires) if not creation_kwargs.get("pyproject_backend") and build_backend: @@ -1708,18 +1666,18 @@ class FileRequirement(object): if name: creation_kwargs["name"] = name cls_inst = cls(**creation_kwargs) # type: ignore - if parsed_line and not cls_inst._parsed_line: - cls_inst._parsed_line = parsed_line - if not cls_inst._parsed_line: - cls_inst._parsed_line = Line(cls_inst.line_part) - if cls_inst._parsed_line and cls_inst.parsed_line.ireq and not cls_inst.parsed_line.ireq.req: - if cls_inst.req: - cls_inst._parsed_line._ireq.req = cls_inst.req + # if parsed_line and not cls_inst._parsed_line: + # cls_inst._parsed_line = parsed_line + # if not cls_inst._parsed_line: + # cls_inst._parsed_line = Line(cls_inst.line_part) + # if cls_inst._parsed_line and cls_inst.parsed_line.ireq and not cls_inst.parsed_line.ireq.req: + # if cls_inst.req: + # cls_inst._parsed_line._ireq.req = cls_inst.req return cls_inst @classmethod def from_line(cls, line, extras=None, parsed_line=None): - # type: (str, Optional[Tuple[str]], Optional[Line]) -> FileRequirement + # type: (Text, Optional[Tuple[Text]], Optional[Line]) -> FileRequirement line = line.strip('"').strip("'") link = None path = None @@ -1770,7 +1728,7 @@ class FileRequirement(object): @classmethod def from_pipfile(cls, name, pipfile): - # type: (str, Dict[str, Any]) -> FileRequirement + # type: (Text, Dict[Text, Any]) -> FileRequirement # Parse the values out. After this dance we should have two variables: # path - Local filesystem path. # uri - Absolute URI that is parsable with urlsplit. @@ -1842,9 +1800,9 @@ class FileRequirement(object): @property def line_part(self): - # type: () -> str - link_url = None # type: Optional[str] - seed = None # type: Optional[str] + # type: () -> Text + link_url = None # type: Optional[Text] + seed = None # type: Optional[Text] if self.link is not None: link_url = unquote(self.link.url_without_fragment) if self._uri_scheme and self._uri_scheme == "path": @@ -1864,7 +1822,7 @@ class FileRequirement(object): @property def pipfile_part(self): - # type: () -> Dict[str, Dict[str, Any]] + # type: () -> Dict[Text, Dict[Text, Any]] excludes = [ "_base_line", "_has_hashed_name", "setup_path", "pyproject_path", "_uri_scheme", "pyproject_requires", "pyproject_backend", "_setup_info", "_parsed_line" @@ -1876,7 +1834,7 @@ class FileRequirement(object): pipfile_dict.pop("_uri_scheme") # For local paths and remote installable artifacts (zipfiles, etc) collision_keys = {"file", "uri", "path"} - collision_order = ["file", "uri", "path"] # type: List[str] + collision_order = ["file", "uri", "path"] # type: List[Text] key_match = next(iter(k for k in collision_order if k in pipfile_dict.keys())) if self._uri_scheme: dict_key = self._uri_scheme @@ -1918,39 +1876,40 @@ class VCSRequirement(FileRequirement): #: Whether the repository is editable editable = attr.ib(default=None) # type: Optional[bool] #: URI for the repository - uri = attr.ib(default=None) # type: Optional[str] + uri = attr.ib(default=None) # type: Optional[Text] #: path to the repository, if it's local - path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) # type: Optional[str] + path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) # type: Optional[Text] #: vcs type, i.e. git/hg/svn - vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) # type: Optional[str] + vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) # type: Optional[Text] #: vcs reference name (branch / commit / tag) - ref = attr.ib(default=None) # type: Optional[str] + ref = attr.ib(default=None) # type: Optional[Text] #: Subdirectory to use for installation if applicable - subdirectory = attr.ib(default=None) # type: Optional[str] - _repo = attr.ib(default=None) # type: Optional['VCSRepository'] - _base_line = attr.ib(default=None) # type: Optional[str] - name = attr.ib() - link = attr.ib() - req = attr.ib() + subdirectory = attr.ib(default=None) # type: Optional[Text] + _repo = attr.ib(default=None) # type: Optional[VCSRepository] + _base_line = attr.ib(default=None) # type: Optional[Text] + name = attr.ib() # type: Text + link = attr.ib() # type: Optional[pip_shims.shims.Link] + req = attr.ib() # type: Optional[RequirementType] def __attrs_post_init__(self): # type: () -> None if not self.uri: if self.path: self.uri = pip_shims.shims.path_to_url(self.path) - split = urllib_parse.urlsplit(self.uri) - scheme, rest = split[0], split[1:] - vcs_type = "" - if "+" in scheme: - vcs_type, scheme = scheme.split("+", 1) - vcs_type = "{0}+".format(vcs_type) - new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",)) - new_uri = "{0}{1}".format(vcs_type, new_uri) - self.uri = new_uri - if self.req and ( - self.parsed_line.ireq and not self.parsed_line.ireq.req - ): - self.parsed_line._ireq.req = self.req + if self.uri is not None: + split = urllib_parse.urlsplit(self.uri) + scheme, rest = split[0], split[1:] + vcs_type = "" + if "+" in scheme: + vcs_type, scheme = scheme.split("+", 1) + vcs_type = "{0}+".format(vcs_type) + new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",)) + new_uri = "{0}{1}".format(vcs_type, new_uri) + self.uri = new_uri + # if self.req and self._parsed_line and ( + # self._parsed_line.ireq and not self._parsed_line.ireq.req + # ): + # self._parsed_line._ireq.req = self.req @link.default def get_link(self): @@ -1968,7 +1927,7 @@ class VCSRequirement(FileRequirement): @name.default def get_name(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] return ( self.link.egg_fragment or self.req.name if getattr(self, "req", None) @@ -1977,7 +1936,7 @@ class VCSRequirement(FileRequirement): @property def vcs_uri(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] uri = self.uri if not any(uri.startswith("{0}+".format(vcs)) for vcs in VCS_LIST): uri = "{0}+{1}".format(self.vcs, uri) @@ -1985,12 +1944,15 @@ class VCSRequirement(FileRequirement): @property def setup_info(self): + if self._parsed_line and self._parsed_line.setup_info: + if not self._parsed_line.setup_info.name: + self._parsed_line._setup_info.get_info() + return self._parsed_line.setup_info if self._repo: from .setup_info import SetupInfo self._setup_info = SetupInfo.from_ireq(Line(self._repo.checkout_directory).ireq) + self._setup_info.get_info() return self._setup_info - if self._parsed_line and self._parsed_line.setup_info: - return self._parsed_line.setup_info ireq = self.parsed_line.ireq from .setup_info import SetupInfo self._setup_info = SetupInfo.from_ireq(ireq) @@ -2057,13 +2019,16 @@ class VCSRequirement(FileRequirement): def repo(self): # type: () -> VCSRepository if self._repo is None: - self._repo = self.get_vcs_repo() - if self._parsed_line: - self._parsed_line.vcsrepo = self._repo + if self._parsed_line and self._parsed_line.vcsrepo: + self._repo = self._parsed_line.vcsrepo + else: + self._repo = self.get_vcs_repo() + if self._parsed_line: + self._parsed_line.vcsrepo = self._repo return self._repo def get_checkout_dir(self, src_dir=None): - # type: (Optional[str]) -> str + # type: (Optional[Text]) -> Text src_dir = os.environ.get("PIP_SRC", None) if not src_dir else src_dir checkout_dir = None if self.is_local: @@ -2079,11 +2044,12 @@ class VCSRequirement(FileRequirement): return checkout_dir return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) - def get_vcs_repo(self, src_dir=None): - # type: (Optional[str]) -> VCSRepository + def get_vcs_repo(self, src_dir=None, checkout_dir=None): + # type: (Optional[Text], Optional[Text]) -> VCSRepository from .vcs import VCSRepository - checkout_dir = self.get_checkout_dir(src_dir=src_dir) + if checkout_dir is None: + checkout_dir = self.get_checkout_dir(src_dir=src_dir) vcsrepo = VCSRepository( url=self.link.url, name=self.name, @@ -2110,13 +2076,13 @@ class VCSRequirement(FileRequirement): return vcsrepo def get_commit_hash(self): - # type: () -> str + # type: () -> Text hash_ = None hash_ = self.repo.get_commit_hash() return hash_ def update_repo(self, src_dir=None, ref=None): - # type: (Optional[str], Optional[str]) -> str + # type: (Optional[Text], Optional[Text]) -> Text if ref: self.ref = ref else: @@ -2131,7 +2097,7 @@ class VCSRequirement(FileRequirement): @contextmanager def locked_vcs_repo(self, src_dir=None): - # type: (Optional[str]) -> Generator[VCSRepository, None, None] + # type: (Optional[Text]) -> Generator[VCSRepository, None, None] if not src_dir: src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") vcsrepo = self.get_vcs_repo(src_dir=src_dir) @@ -2153,12 +2119,11 @@ class VCSRequirement(FileRequirement): if self._parsed_line: self._parsed_line.vcsrepo = vcsrepo if self._setup_info: - self._setup_info._requirements = () - self._setup_info._extras_requirements = () - self._setup_info.build_requires = () - self._setup_info.setup_requires = () - self._setup_info.version = None - self._setup_info.metadata = None + _old_setup_info = self._setup_info + self._setup_info = attr.evolve( + self._setup_info, requirements=(), _extras_requirements=(), + build_requires=(), setup_requires=(), version=None, metadata=None + ) if self.parsed_line: self._parsed_line.vcsrepo = vcsrepo # self._parsed_line._specifier = "=={0}".format(self.setup_info.version) @@ -2169,10 +2134,11 @@ class VCSRequirement(FileRequirement): yield vcsrepo finally: self._repo = orig_repo + # self._setup_info = _old_setup_info @classmethod def from_pipfile(cls, name, pipfile): - # type: (str, Dict[str, Union[List[str], str, bool]]) -> VCSRequirement + # type: (Text, Dict[Text, Union[List[Text], Text, bool]]) -> VCSRequirement creation_args = {} pipfile_keys = [ k @@ -2216,26 +2182,26 @@ class VCSRequirement(FileRequirement): creation_args[key] = pipfile.get(key) creation_args["name"] = name cls_inst = cls(**creation_args) - if cls_inst._parsed_line is None: - vcs_uri = build_vcs_uri( - vcs=cls_inst.vcs, uri=add_ssh_scheme_to_git_uri(cls_inst.uri), - name=cls_inst.name, ref=cls_inst.ref, subdirectory=cls_inst.subdirectory, - extras=cls_inst.extras - ) - if cls_inst.editable: - vcs_uri = "-e {0}".format(vcs_uri) - cls_inst._parsed_line = Line(vcs_uri) - if not cls_inst.name and cls_inst._parsed_line.name: - cls_inst.name = cls_inst._parsed_line.name - if cls_inst.req and ( - cls_inst._parsed_line.ireq and not cls_inst.parsed_line.ireq.req - ): - cls_inst._parsed_line.ireq.req = cls_inst.req + # if cls_inst._parsed_line is None: + # vcs_uri = build_vcs_uri( + # vcs=cls_inst.vcs, uri=add_ssh_scheme_to_git_uri(cls_inst.uri), + # name=cls_inst.name, ref=cls_inst.ref, subdirectory=cls_inst.subdirectory, + # extras=cls_inst.extras + # ) + # if cls_inst.editable: + # vcs_uri = "-e {0}".format(vcs_uri) + # cls_inst._parsed_line = Line(vcs_uri) + # if not cls_inst.name and cls_inst._parsed_line.name: + # cls_inst.name = cls_inst._parsed_line.name + # if cls_inst.req and ( + # cls_inst._parsed_line.ireq and not cls_inst.parsed_line.ireq.req + # ): + # cls_inst._parsed_line.ireq.req = cls_inst.req return cls_inst @classmethod def from_line(cls, line, editable=None, extras=None, parsed_line=None): - # type: (str, Optional[bool], Optional[Tuple[str]], Optional[Line]) -> VCSRequirement + # type: (Text, Optional[bool], Optional[Tuple[Text]], Optional[Line]) -> VCSRequirement relpath = None if parsed_line is None: parsed_line = Line(line) @@ -2313,7 +2279,7 @@ class VCSRequirement(FileRequirement): @property def line_part(self): - # type: () -> str + # type: () -> Text """requirements.txt compatible line part sans-extras""" if self.is_local: base_link = self.link @@ -2344,7 +2310,7 @@ class VCSRequirement(FileRequirement): @staticmethod def _choose_vcs_source(pipfile): - # type: (Dict[str, Union[List[str], str, bool]]) -> Dict[str, Union[List[str], str, bool]] + # type: (Dict[Text, Union[List[Text], Text, bool]]) -> Dict[Text, Union[List[Text], Text, bool]] src_keys = [k for k in pipfile.keys() if k in ["path", "uri", "file"]] if src_keys: chosen_key = first(src_keys) @@ -2357,7 +2323,7 @@ class VCSRequirement(FileRequirement): @property def pipfile_part(self): - # type: () -> Dict[str, Dict[str, Union[List[str], str, bool]]] + # type: () -> Dict[Text, Dict[Text, Union[List[Text], Text, bool]]] excludes = [ "_repo", "_base_line", "setup_path", "_has_hashed_name", "pyproject_path", "pyproject_requires", "pyproject_backend", "_setup_info", "_parsed_line", @@ -2373,15 +2339,15 @@ class VCSRequirement(FileRequirement): @attr.s(cmp=True, hash=True) class Requirement(object): - name = attr.ib(cmp=True) # type: str - vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs), cmp=True) # type: Optional[str] + name = attr.ib(cmp=True) # type: Text + vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs), cmp=True) # type: Optional[Text] req = attr.ib(default=None, cmp=True) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] - markers = attr.ib(default=None, cmp=True) # type: Optional[str] - _specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers), cmp=True) # type: Optional[str] - index = attr.ib(default=None, cmp=True) # type: Optional[str] + markers = attr.ib(default=None, cmp=True) # type: Optional[Text] + _specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers), cmp=True) # type: Optional[Text] + index = attr.ib(default=None, cmp=True) # type: Optional[Text] editable = attr.ib(default=None, cmp=True) # type: Optional[bool] - hashes = attr.ib(factory=frozenset, converter=frozenset, cmp=True) # type: Optional[Tuple[str]] - extras = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Optional[Tuple[str]] + hashes = attr.ib(factory=frozenset, converter=frozenset, cmp=True) # type: Optional[Tuple[Text]] + extras = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Optional[Tuple[Text]] abstract_dep = attr.ib(default=None, cmp=False) # type: Optional[AbstractDependency] _line_instance = attr.ib(default=None, cmp=False) # type: Optional[Line] _ireq = attr.ib(default=None, cmp=False) # type: Optional[pip_shims.InstallRequirement] @@ -2391,7 +2357,7 @@ class Requirement(object): @name.default def get_name(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] return self.req.name @property @@ -2408,7 +2374,7 @@ class Requirement(object): return attr.evolve(self, hashes=frozenset(new_hashes)) def get_hashes_as_pip(self, as_list=False): - # type: () -> Union[str, List[str]] + # type: (bool) -> Union[Text, List[Text]] if self.hashes: if as_list: return [HASH_STRING.format(h) for h in self.hashes] @@ -2417,12 +2383,12 @@ class Requirement(object): @property def hashes_as_pip(self): - # type: () -> Union[str, List[str]] + # type: () -> Union[Text, List[Text]] self.get_hashes_as_pip() @property def markers_as_pip(self): - # type: () -> str + # type: () -> Text if self.markers: return " ; {0}".format(self.markers).replace('"', "'") @@ -2430,7 +2396,7 @@ class Requirement(object): @property def extras_as_pip(self): - # type: () -> str + # type: () -> Text if self.extras: return "[{0}]".format( ",".join(sorted([extra.lower() for extra in self.extras])) @@ -2440,7 +2406,7 @@ class Requirement(object): @property def commit_hash(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if not self.is_vcs: return None commit_hash = None @@ -2450,7 +2416,7 @@ class Requirement(object): @_specifiers.default def get_specifiers(self): - # type: () -> Optional[str] + # type: () -> Text if self.req and self.req.req and self.req.req.specifier: return specs_to_string(self.req.req.specifier) return "" @@ -2493,12 +2459,22 @@ class Requirement(object): self._specifiers if include_specifiers else "", self.markers_as_pip, ] - self._line_instance = Line("".join(parts)) + line = "".join(parts) + if line is None: + return None + self._line_instance = Line(line) return self._line_instance + @line_instance.setter + def line_instance(self, line_instance): + # type: (Line) -> None + if self.req and not self.req._parsed_line: + self.req._parsed_line = line_instance + self._line_instance = line_instance + @property def specifiers(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self._specifiers: return self._specifiers else: @@ -2506,10 +2482,11 @@ class Requirement(object): if specs: self._specifiers = specs return specs - if not self._specifiers and self.req and self.req.req and self.req.req.specifier: - self._specifiers = specs_to_string(self.req.req.specifier) - elif self.is_named and not self._specifiers: + if self.is_named and not self._specifiers: self._specifiers = self.req.version + elif not self.editable and not self.is_named: + if self.line_instance and self.line_instance.setup_info and self.line_instance.setup_info.version: + self._specifiers = "=={0}".format(self.req.setup_info.version) elif self.req.parsed_line.specifiers and not self._specifiers: self._specifiers = specs_to_string(self.req.parsed_line.specifiers) elif self.line_instance.specifiers and not self._specifiers: @@ -2538,7 +2515,7 @@ class Requirement(object): @property def build_backend(self): - # type: () -> Optional[str] + # type: () -> Optional[Text] if self.is_vcs or (self.is_file_or_url and self.req.is_local): setup_info = self.run_requires() build_backend = setup_info.get("build_backend") @@ -2571,6 +2548,7 @@ class Requirement(object): @property def normalized_name(self): + # type: () -> Text return canonicalize_name(self.name) def copy(self): @@ -2578,7 +2556,7 @@ class Requirement(object): @classmethod def from_line(cls, line): - # type: (str) -> Requirement + # type: (Text) -> Requirement if isinstance(line, pip_shims.shims.InstallRequirement): line = format_requirement(line) parsed_line = Line(line) @@ -2593,73 +2571,11 @@ class Requirement(object): ) else: r = named_req_from_parsed_line(parsed_line) - # hashes = None - # if "--hash=" in line: - # hashes = line.split(" --hash=") - # line, hashes = hashes[0], hashes[1:] - # editable = line.startswith("-e ") - # line = line.split(" ", 1)[1] if editable else line - # line, markers = split_markers_from_line(line) - # line, extras = pip_shims.shims._strip_extras(line) - # if extras: - # extras = tuple(parse_extras(extras)) - # line = line.strip('"').strip("'").strip() - # line_with_prefix = "-e {0}".format(line) if editable else line - # vcs = None - # # Installable local files and installable non-vcs urls are handled - # # as files, generally speaking - # line_is_vcs = is_vcs(line) - # is_direct_url = False - # # check for pep-508 compatible requirements - # name, _, possible_url = line.partition("@") - # name = name.strip() - # if possible_url is not None: - # possible_url = possible_url.strip() - # is_direct_url = is_valid_url(possible_url) - # if not line_is_vcs: - # line_is_vcs = is_vcs(possible_url) - # if is_installable_file(line) or ( - # (is_valid_url(possible_url) or is_file_url(line) or is_valid_url(line)) and - # not (line_is_vcs or is_vcs(possible_url)) - # ): - # r = FileRequirement.from_line(line_with_prefix, extras=extras, parsed_line=parsed_line) - # elif line_is_vcs: - # r = VCSRequirement.from_line(line_with_prefix, extras=extras, parsed_line=parsed_line) - # if isinstance(r, VCSRequirement): - # vcs = r.vcs - # elif line == "." and not is_installable_file(line): - # raise RequirementError( - # "Error parsing requirement %s -- are you sure it is installable?" % line - # ) - # else: - # specs = "!=<>~" - # spec_matches = set(specs) & set(line) - # version = None - # name = "{0}".format(line) - # if spec_matches: - # spec_idx = min((line.index(match) for match in spec_matches)) - # name = line[:spec_idx] - # version = line[spec_idx:] - # if not extras: - # name, extras = pip_shims.shims._strip_extras(name) - # if extras: - # extras = tuple(parse_extras(extras)) - # if version: - # name = "{0}{1}".format(name, version) - # r = NamedRequirement.from_line(line, parsed_line=parsed_line) req_markers = None if parsed_line.markers: req_markers = PackagingRequirement("fakepkg; {0}".format(parsed_line.markers)) if r is not None and r.req is not None: r.req.marker = getattr(req_markers, "marker", None) if req_markers else None - # r.req.local_file = getattr(r.req, "local_file", False) - # name = getattr(r, "name", None) - # if name is None and getattr(r.req, "name", None) is not None: - # name = r.req.name - # elif name is None and getattr(r.req, "key", None) is not None: - # name = r.req.key - # if name is not None and getattr(r.req, "name", None) is None: - # r.req.name = name args = { "name": r.name, "vcs": parsed_line.vcs, @@ -2716,10 +2632,12 @@ class Requirement(object): if r.req is not None: r.req.marker = req_markers.marker extras = _pipfile.get("extras") - r.req.specifier = SpecifierSet(_pipfile["version"]) - r.req.extras = ( - tuple(sorted(dedup([extra.lower() for extra in extras]))) if extras else () - ) + if r.req: + if r.req.specifier: + r.req.specifier = SpecifierSet(_pipfile["version"]) + r.req.extras = ( + tuple(sorted(dedup([extra.lower() for extra in extras]))) if extras else () + ) args = { "name": r.name, "vcs": vcs, @@ -2732,17 +2650,6 @@ class Requirement(object): if any(key in _pipfile for key in ["hash", "hashes"]): args["hashes"] = _pipfile.get("hashes", [pipfile.get("hash")]) cls_inst = cls(**args) - if not cls_inst.req._parsed_line: - parsed_line = Line(cls_inst.as_line()) - cls_inst.req._parsed_line = parsed_line - if not cls_inst.line_instance: - cls_inst.line_instance = parsed_line - if not cls_inst.is_named and not cls_inst.req._setup_info and parsed_line.setup_info: - cls_inst.req._setup_info = parsed_line.setup_info - if not cls_inst.req.name and parsed_line.setup_info.name: - cls_inst.name = cls_inst.req.name = parsed_line.setup_info.name - if not cls_inst.req.name and parsed_line.name: - cls_inst.name = cls_inst.req.name = parsed_line.name return cls_inst def as_line( @@ -2803,7 +2710,7 @@ class Requirement(object): markers = self.markers if markers: fake_pkg = PackagingRequirement("fakepkg; {0}".format(markers)) - markers = fake_pkg.markers + markers = fake_pkg.marker return markers def get_specifier(self): @@ -3073,7 +2980,7 @@ def named_req_from_parsed_line(parsed_line): # type: (Line) -> NamedRequirement return NamedRequirement( name=parsed_line.name, - version=parsed_line.specifier, # type: Optional[str] + version=parsed_line.specifier, req=parsed_line.requirement, extras=parsed_line.extras, editable=parsed_line.editable, diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 31134b03..39eaa57f 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -15,11 +15,13 @@ import pep517.envbuild import pep517.wrappers import six from appdirs import user_cache_dir +from distlib.wheel import Wheel +from packaging.markers import Marker from six.moves import configparser from six.moves.urllib.parse import unquote, urlparse, urlunparse from vistir.compat import Iterable, Path -from vistir.contextmanagers import cd, temp_path, replaced_streams +from vistir.contextmanagers import cd, temp_path from vistir.misc import run from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p, rmtree @@ -46,9 +48,15 @@ except ImportError: if MYPY_RUNNING: - from typing import Any, Dict, List, Generator, Optional, Union, Tuple - from pip_shims.shims import InstallRequirement - from pkg_resources import Requirement as PkgResourcesRequirement + from typing import Any, Dict, List, Generator, Optional, Union, Tuple, TypeVar, Text, Set + from pip_shims.shims import InstallRequirement, PackageFinder + from pkg_resources import ( + PathMetadata, DistInfoDistribution, Requirement as PkgResourcesRequirement + ) + from packaging.requirements import Requirement as PackagingRequirement + TRequirement = TypeVar("TRequirement") + RequirementType = TypeVar('RequirementType', covariant=True, bound=PackagingRequirement) + MarkerType = TypeVar('MarkerType', covariant=True, bound=Marker) CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) @@ -59,9 +67,17 @@ _setup_stop_after = None _setup_distribution = None +class BuildEnv(pep517.envbuild.BuildEnvironment): + def pip_install(self, reqs): + cmd = [sys.executable, '-m', 'pip', 'install', '--ignore-installed', '--prefix', + self.path] + list(reqs) + run(cmd, block=True, combine_stderr=True, return_object=False, + write_to_stdout=False, nospin=True) + + @contextlib.contextmanager def _suppress_distutils_logs(): - # type: () -> None + # type: () -> Generator[None, None, None] """Hack to hide noise generated by `setup.py develop`. There isn't a good way to suppress them now, so let's monky-patch. @@ -81,7 +97,7 @@ def _suppress_distutils_logs(): @ensure_mkdir_p(mode=0o775) def _get_src_dir(root): - # type: (str) -> str + # type: (Text) -> Text src = os.environ.get("PIP_SRC") if src: return src @@ -97,7 +113,7 @@ def _get_src_dir(root): def ensure_reqs(reqs): - # type: (List[Union[str, PkgResourcesRequirement]]) -> List[PkgResourcesRequirement] + # type: (List[Union[Text, PkgResourcesRequirement]]) -> List[PkgResourcesRequirement] import pkg_resources if not isinstance(reqs, Iterable): raise TypeError("Expecting an Iterable, got %r" % reqs) @@ -107,40 +123,43 @@ def ensure_reqs(reqs): continue if isinstance(req, six.string_types): req = pkg_resources.Requirement.parse("{0}".format(str(req))) - req = strip_extras_markers_from_requirement(req) + # req = strip_extras_markers_from_requirement(req) new_reqs.append(req) return new_reqs +def pep517_subprocess_runner(cmd, cwd=None, extra_environ=None): + # type: (List[Text], Optional[Text], Optional[Dict[Text, Text]]) -> None + """The default method of calling the wrapper subprocess.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + run(cmd, cwd=cwd, env=env, block=True, combine_stderr=True, return_object=False, + write_to_stdout=False, nospin=True) + + def _prepare_wheel_building_kwargs(ireq=None, src_root=None, src_dir=None, editable=False): - # type: (Optional[InstallRequirement], Optional[str], Optional[str], bool) -> Dict[str, str] - download_dir = os.path.join(CACHE_DIR, "pkgs") # type: str + # type: (Optional[InstallRequirement], Optional[Text], Optional[Text], bool) -> Dict[Text, Text] + download_dir = os.path.join(CACHE_DIR, "pkgs") # type: Text mkdir_p(download_dir) - wheel_download_dir = os.path.join(CACHE_DIR, "wheels") # type: str + wheel_download_dir = os.path.join(CACHE_DIR, "wheels") # type: Text mkdir_p(wheel_download_dir) if src_dir is None: if editable and src_root is not None: src_dir = src_root elif ireq is None and src_root is not None: - src_dir = _get_src_dir(root=src_root) # type: str - # # elif ireq is not None and ireq.editable is not None and ireq.source_dir is not None: - # # src_dir = ireq.source_dir + src_dir = _get_src_dir(root=src_root) # type: Text elif ireq is not None and ireq.editable and src_root is not None: src_dir = _get_src_dir(root=src_root) else: src_dir = create_tracked_tempdir(prefix="reqlib-src") - # This logic matches pip's behavior, although I don't fully understand the - # intention. I guess the idea is to build editables in-place, otherwise out - # of the source tree? - # if (ireq is not None and ireq.editable) or editable: - # build_dir = src_dir - # else: - # Let's always resolve in isolation - # src_dir = create_tracked_tempdir(prefix="reqlib-src") + if src_dir is None: + src_dir = create_tracked_tempdir(prefix="reqlib-src") build_dir = create_tracked_tempdir(prefix="reqlib-build") return { @@ -152,7 +171,7 @@ def _prepare_wheel_building_kwargs(ireq=None, src_root=None, src_dir=None, edita def iter_metadata(path, pkg_name=None, metadata_type="egg-info"): - # type: (str, Optional[str], str) -> Generator + # type: (Text, Optional[Text], Text) -> Generator if pkg_name is not None: pkg_variants = get_name_variants(pkg_name) non_matching_dirs = [] @@ -170,7 +189,7 @@ def iter_metadata(path, pkg_name=None, metadata_type="egg-info"): def find_egginfo(target, pkg_name=None): - # type: (str, Optional[str]) -> Generator + # type: (Text, Optional[Text]) -> Generator egg_dirs = ( egg_dir for egg_dir in iter_metadata(target, pkg_name=pkg_name) if egg_dir is not None @@ -183,7 +202,7 @@ def find_egginfo(target, pkg_name=None): def find_distinfo(target, pkg_name=None): - # type: (str, Optional[str]) -> Generator + # type: (Text, Optional[Text]) -> Generator dist_dirs = ( dist_dir for dist_dir in iter_metadata(target, pkg_name=pkg_name, metadata_type="dist-info") if dist_dir is not None @@ -195,10 +214,18 @@ def find_distinfo(target, pkg_name=None): yield dist_dir -def get_metadata(path, pkg_name=None): +def get_metadata(path, pkg_name=None, metadata_type=None): + # type: (Text, Optional[Text], Optional[Text]) -> Dict[Text, Union[Text, List[RequirementType], Dict[Text, RequirementType]]] + metadata_dirs = [] + wheel_allowed = metadata_type == "wheel" or metadata_type is None + egg_allowed = metadata_type == "egg" or metadata_type is None egg_dir = next(iter(find_egginfo(path, pkg_name=pkg_name)), None) dist_dir = next(iter(find_distinfo(path, pkg_name=pkg_name)), None) - matched_dir = next(iter(d for d in (dist_dir, egg_dir) if d is not None), None) + if dist_dir and wheel_allowed: + metadata_dirs.append(dist_dir) + if egg_dir and egg_allowed: + metadata_dirs.append(egg_dir) + matched_dir = next(iter(d for d in metadata_dirs if d is not None), None) metadata_dir = None base_dir = None if matched_dir is not None: @@ -208,72 +235,126 @@ def get_metadata(path, pkg_name=None): dist = None distinfo_dist = None egg_dist = None - if dist_dir is not None: + if wheel_allowed and dist_dir is not None: distinfo_dist = next(iter(pkg_resources.find_distributions(base_dir)), None) - if egg_dir is not None: + if egg_allowed and egg_dir is not None: path_metadata = pkg_resources.PathMetadata(base_dir, metadata_dir) egg_dist = next( iter(pkg_resources.distributions_from_metadata(path_metadata.egg_info)), None, ) dist = next(iter(d for d in (distinfo_dist, egg_dist) if d is not None), None) - if dist: - try: - requires = dist.requires() - except Exception: - requires = [] - try: - dep_map = dist._build_dep_map() - except Exception: - dep_map = {} - deps = [] - extras = {} - for k in dep_map.keys(): - if k is None: - deps.extend(dep_map.get(k)) - continue - else: - extra = None - _deps = dep_map.get(k) - if k.startswith(":python_version"): - marker = k.replace(":", "; ") - else: - marker = "" - extra = "{0}".format(k) - _deps = ["{0}{1}".format(str(req), marker) for req in _deps] - _deps = ensure_reqs(_deps) - if extra: - extras[extra] = _deps - else: - deps.extend(_deps) - return { - "name": dist.project_name, - "version": dist.version, - "requires": requires, - "extras": extras - } + if dist is not None: + return get_metadata_from_dist(dist) + return {} + + +def get_extra_name_from_marker(marker): + # type: (MarkerType) -> Optional[Text] + if not marker: + raise ValueError("Invalid value for marker: {0!r}".format(marker)) + if not getattr(marker, "_markers", None): + raise TypeError("Expecting a marker instance, received {0!r}".format(marker)) + for elem in marker._markers: + if isinstance(elem, tuple) and elem[0].value == "extra": + return elem[2].value + return None + + +def get_metadata_from_wheel(wheel_path): + # type: (Text) -> Dict[Any, Any] + if not isinstance(wheel_path, six.string_types): + raise TypeError("Expected string instance, received {0!r}".format(wheel_path)) + try: + dist = Wheel(wheel_path) + except Exception: + pass + metadata = dist.metadata + name = metadata.name + version = metadata.version + requires = [] + extras_keys = getattr(metadata, "extras", None) + extras = {} + for req in getattr(metadata, "run_requires", []): + parsed_req = init_requirement(req) + parsed_marker = parsed_req.marker + if parsed_marker: + extra = get_extra_name_from_marker(parsed_marker) + if extra is None: + requires.append(parsed_req) + continue + if extra not in extras: + extras[extra] = [] + parsed_req = strip_extras_markers_from_requirement(parsed_req) + extras[extra].append(parsed_req) + else: + requires.append(parsed_req) + return { + "name": name, + "version": version, + "requires": requires, + "extras": extras + } + + +def get_metadata_from_dist(dist): + # type: (Union[PathMetadata, DistInfoDistribution]) -> Dict[Text, Union[Text, List[RequirementType], Dict[Text, RequirementType]]] + try: + requires = dist.requires() + except Exception: + requires = [] + try: + dep_map = dist._build_dep_map() + except Exception: + dep_map = {} + deps = [] + extras = {} + for k in dep_map.keys(): + if k is None: + deps.extend(dep_map.get(k)) + continue + else: + extra = None + _deps = dep_map.get(k) + if k.startswith(":python_version"): + marker = k.replace(":", "; ") + else: + marker = "" + extra = "{0}".format(k) + _deps = ["{0}{1}".format(str(req), marker) for req in _deps] + _deps = ensure_reqs(_deps) + if extra: + extras[extra] = _deps + else: + deps.extend(_deps) + return { + "name": dist.project_name, + "version": dist.version, + "requires": requires, + "extras": extras + } @attr.s(slots=True, frozen=True) class BaseRequirement(object): - name = attr.ib(type=str, default="", cmp=True) + name = attr.ib(default="", cmp=True) # type: Text requirement = attr.ib(default=None, cmp=True) # type: Optional[PkgResourcesRequirement] def __str__(self): - # type: () -> str + # type: () -> Text return "{0}".format(str(self.requirement)) def as_dict(self): - # type: () -> Dict[str, Optional[PkgResourcesRequirement]] + # type: () -> Dict[Text, Optional[PkgResourcesRequirement]] return {self.name: self.requirement} def as_tuple(self): - # type: () -> Tuple[str, Optional[PkgResourcesRequirement]] + # type: () -> Tuple[Text, Optional[PkgResourcesRequirement]] return (self.name, self.requirement) @classmethod def from_string(cls, line): - # type: (str) -> BaseRequirement + # type: (Text) -> BaseRequirement line = line.strip() req = init_requirement(line) return cls.from_req(req) @@ -294,11 +375,11 @@ class BaseRequirement(object): @attr.s(slots=True, frozen=True) class Extra(object): - name = attr.ib(type=str, default=None, cmp=True) + name = attr.ib(default=None, cmp=True) # type: Text requirements = attr.ib(factory=frozenset, cmp=True, type=frozenset) def __str__(self): - # type: () -> str + # type: () -> Text return "{0}: {{{1}}}".format(self.section, ", ".join([r.name for r in self.requirements])) def add(self, req): @@ -308,34 +389,36 @@ class Extra(object): return self def as_dict(self): - # type: () -> Dict[str, Tuple[PkgResourcesRequirement]] + # type: () -> Dict[Text, Tuple[RequirementType, ...]] return {self.name: tuple([r.requirement for r in self.requirements])} @attr.s(slots=True, cmp=True, hash=True) class SetupInfo(object): - name = attr.ib(type=str, default=None, cmp=True) - base_dir = attr.ib(type=str, default=None, cmp=True, hash=False) - version = attr.ib(type=str, default=None, cmp=True) + name = attr.ib(default=None, cmp=True) # type: Text + base_dir = attr.ib(default=None, cmp=True, hash=False) # type: Text + version = attr.ib(default=None, cmp=True) # type: Text _requirements = attr.ib(type=frozenset, factory=frozenset, cmp=True, hash=True) build_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) - build_backend = attr.ib(type=str, default="setuptools.build_meta:__legacy__", cmp=True) + build_backend = attr.ib(default="setuptools.build_meta:__legacy__", cmp=True) # type: Text setup_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None, cmp=True) _extras_requirements = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) setup_cfg = attr.ib(type=Path, default=None, cmp=True, hash=False) setup_py = attr.ib(type=Path, default=None, cmp=True, hash=False) pyproject = attr.ib(type=Path, default=None, cmp=True, hash=False) - ireq = attr.ib(default=None, cmp=True, hash=False) + ireq = attr.ib(default=None, cmp=True, hash=False) # type: Optional[InstallRequirement] extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict, cmp=False, hash=False) - metadata = attr.ib(default=None, type=tuple) + metadata = attr.ib(default=None) # type: Optional[Tuple[Text]] @property def requires(self): + # type: () -> Dict[Text, RequirementType] return {req.name: req.requirement for req in self._requirements} @property def extras(self): + # type: () -> Dict[Text, Optional[Any]] extras_dict = {} extras = set(self._extras_requirements) for section, deps in extras: @@ -347,6 +430,7 @@ class SetupInfo(object): @classmethod def get_setup_cfg(cls, setup_cfg_path): + # type: (Text) -> Dict[Text, Union[Text, None, Set[BaseRequirement], List[Text], Tuple[Text, Tuple[BaseRequirement]]]] if os.path.exists(setup_cfg_path): default_opts = { "metadata": {"name": "", "version": ""}, @@ -365,7 +449,7 @@ class SetupInfo(object): results["name"] = parser.get("metadata", "name") if parser.has_option("metadata", "version"): results["version"] = parser.get("metadata", "version") - install_requires = set() + install_requires = set() # type: Set[BaseRequirement] if parser.has_option("options", "install_requires"): install_requires = set([ BaseRequirement.from_string(dep) @@ -395,7 +479,7 @@ class SetupInfo(object): @property def egg_base(self): - base = None # type: Optional[Path] + base = None # type: Optional[Text] if self.setup_py.exists(): base = self.setup_py.parent elif self.pyproject.exists(): @@ -405,7 +489,7 @@ class SetupInfo(object): if base is None: base = Path(self.base_dir) if base is None: - base = Path(self.extra_kwargs["build_dir"]) + base = Path(self.extra_kwargs["src_dir"]) egg_base = base.joinpath("reqlib-metadata") if not egg_base.exists(): atexit.register(rmtree, egg_base.as_posix()) @@ -413,6 +497,7 @@ class SetupInfo(object): return egg_base.as_posix() def parse_setup_cfg(self): + # type: () -> None if self.setup_cfg is not None and self.setup_cfg.exists(): parsed = self.get_setup_cfg(self.setup_cfg.as_posix()) if self.name is None: @@ -423,7 +508,7 @@ class SetupInfo(object): if self.build_requires: self.build_requires = tuple(set(self.build_requires) | set(build_requires)) self._requirements = frozenset( - set(self._requirements) | parsed["install_requires"] + set(self._requirements) | set(parsed["install_requires"]) ) if self.python_requires is None: self.python_requires = parsed.get("python_requires") @@ -438,6 +523,7 @@ class SetupInfo(object): self._extras_requirements += ((extra, extras_tuple),) def run_setup(self): + # type: () -> None if self.setup_py is not None and self.setup_py.exists(): target_cwd = self.setup_py.parent.as_posix() with temp_path(), cd(target_cwd), _suppress_distutils_logs(): @@ -508,36 +594,65 @@ class SetupInfo(object): if not self.version: self.version = dist.get_version() - def run_pep517(self, build=False): - # type: () -> str - with pep517.envbuild.BuildEnvironment(): - hookcaller = pep517.wrappers.Pep517HookCaller( - self.base_dir, self.build_backend - ) - build_deps = hookcaller.get_requires_for_build_wheel() - if self.ireq.editable: - build_deps += hookcaller.get_requires_for_build_sdist() - metadata_dirname = hookcaller.prepare_metadata_for_build_wheel(self.egg_base) + @contextlib.contextmanager + def run_pep517(self): + # type: (bool) -> Generator[pep517.wrappers.Pep517HookCaller, None, None] + config = {} + config.setdefault("--global-option", []) + builder = pep517.wrappers.Pep517HookCaller( + self.base_dir, self.build_backend + ) + builder._subprocess_runner = pep517_subprocess_runner + with BuildEnv() as env: + env.pip_install(self.build_requires) + try: + reqs = builder.get_requires_for_build_wheel(config_settings=config) + env.pip_install(reqs) + metadata_dirname = builder.prepare_metadata_for_build_wheel( + self.egg_base, config_settings=config + ) + except Exception: + reqs = builder.get_requires_for_build_sdist(config_settings=config) + env.pip_install(reqs) metadata_dir = os.path.join(self.egg_base, metadata_dirname) + yield builder - if build: - return self.build_pep517() - return metadata_dir + def build(self): + # type: () -> Optional[Text] + dist_path = None + with self.run_pep517() as hookcaller: + dist_path = self.build_pep517(hookcaller) + if os.path.exists(os.path.join(self.extra_kwargs["build_dir"], dist_path)): + self.get_metadata_from_wheel( + os.path.join(self.extra_kwargs["build_dir"], dist_path) + ) + if not self.metadata or not self.name: + self.get_egg_metadata() + else: + return dist_path + if not self.metadata or not self.name: + hookcaller._subprocess_runner( + ["setup.py", "egg_info", "--egg-base", self.egg_base] + ) + self.get_egg_metadata() + return dist_path def build_pep517(self, hookcaller): - # type: (pep517.wrappers.Pep517HookCaller) -> Optional[str] + # type: (pep517.wrappers.Pep517HookCaller) -> Optional[Text] dist_path = None try: dist_path = hookcaller.build_wheel( self.extra_kwargs["build_dir"], metadata_directory=self.egg_base ) + return dist_path except Exception: dist_path = hookcaller.build_sdist(self.extra_kwargs["build_dir"]) + self.get_egg_metadata(metadata_type="egg") return dist_path def reload(self): - # type: () -> None + # type: () -> Dict[Text, Any] """ Wipe existing distribution info metadata for rebuilding. """ @@ -548,38 +663,58 @@ class SetupInfo(object): self._extras_requirements = () self.get_info() - def get_egg_metadata(self): + def get_metadata_from_wheel(self, wheel_path): + # type: (Text) -> Dict[Any, Any] + metadata_dict = get_metadata_from_wheel(wheel_path) + if metadata_dict: + self.populate_metadata(metadata_dict) + + def get_egg_metadata(self, metadata_dir=None, metadata_type=None): + # type: (Optional[Text], Optional[Text]) -> None package_indicators = [self.pyproject, self.setup_py, self.setup_cfg] # if self.setup_py is not None and self.setup_py.exists(): + metadata_dirs = [] if any([fn is not None and fn.exists() for fn in package_indicators]): - metadata = get_metadata(self.egg_base, pkg_name=self.name) - if metadata: - self.metadata = tuple([(k, v) for k, v in metadata.items()]) - if self.name is None: - self.name = metadata.get("name", self.name) - if not self.version: - self.version = metadata.get("version", self.version) - self._requirements = frozenset( - set(self._requirements) | set([ + metadata_dirs = [self.extra_kwargs["build_dir"], self.egg_base, self.extra_kwargs["src_dir"]] + if metadata_dir is not None: + metadata_dirs = [metadata_dir] + metadata_dirs + metadata = [ + get_metadata(d, pkg_name=self.name, metadata_type=metadata_type) + for d in metadata_dirs if os.path.exists(d) + ] + metadata = next(iter(d for d in metadata if d is not None), None) + if metadata is not None: + self.populate_metadata(metadata) + + def populate_metadata(self, metadata): + # type: (Dict[Any, Any]) -> None + self.metadata = tuple([(k, v) for k, v in metadata.items()]) + if self.name is None: + self.name = metadata.get("name", self.name) + if not self.version: + self.version = metadata.get("version", self.version) + self._requirements = frozenset( + set(self._requirements) | set([ + BaseRequirement.from_req(req) + for req in metadata.get("requires", []) + ]) + ) + if getattr(self.ireq, "extras", None): + for extra in self.ireq.extras: + extras = metadata.get("extras", {}).get(extra, []) + if extras: + extras_tuple = tuple([ BaseRequirement.from_req(req) - for req in metadata.get("requires", []) + for req in ensure_reqs(extras) + if req is not None ]) - ) - if getattr(self.ireq, "extras", None): - for extra in self.ireq.extras: - extras = metadata.get("extras", {}).get(extra, []) - if extras: - extras_tuple = tuple([ - BaseRequirement.from_req(req) - for req in ensure_reqs(extras) - if req is not None - ]) - self._extras_requirements += ((extra, extras_tuple),) - self._requirements = frozenset( - set(self._requirements) | set(extras_tuple) - ) + self._extras_requirements += ((extra, extras_tuple),) + self._requirements = frozenset( + set(self._requirements) | set(extras_tuple) + ) def run_pyproject(self): + # type: () -> None if self.pyproject and self.pyproject.exists(): result = get_pyproject(self.pyproject.parent) if result is not None: @@ -588,27 +723,25 @@ class SetupInfo(object): self.build_backend = backend else: self.build_backend = "setuptools.build_meta:__legacy__" + if requires: + self.build_requires = tuple(set(requires) | set(self.build_requires)) + else: self.build_requires = ("setuptools", "wheel") - if requires and not self.build_requires: - self.build_requires = tuple(requires) def get_info(self): + # type: () -> Dict[Text, Any] if self.setup_cfg and self.setup_cfg.exists(): with cd(self.base_dir): self.parse_setup_cfg() - if self.pyproject and self.pyproject.exists(): - with cd(self.base_dir), replaced_streams(): - self.run_pyproject() - self.run_pep517() - self.get_egg_metadata() + with cd(self.base_dir): + self.run_pyproject() + self.build() if self.setup_py and self.setup_py.exists() and self.metadata is None: if not self.requires or not self.name: try: with cd(self.base_dir): - for metadata_dir in os.listdir(self.egg_base): - shutil.rmtree(metadata_dir, ignore_errors=True) self.run_setup() except Exception: with cd(self.base_dir): @@ -620,7 +753,7 @@ class SetupInfo(object): return self.as_dict() def as_dict(self): - # type: () -> Dict[str, Any] + # type: () -> Dict[Text, Any] prop_dict = { "name": self.name, "version": self.version, @@ -641,12 +774,14 @@ class SetupInfo(object): @classmethod def from_requirement(cls, requirement, finder=None): + # type: (TRequirement, Optional[PackageFinder]) -> Optional[SetupInfo] ireq = requirement.as_ireq() subdir = getattr(requirement.req, "subdirectory", None) return cls.from_ireq(ireq, subdir=subdir, finder=finder) @classmethod def from_ireq(cls, ireq, subdir=None, finder=None): + # type: (InstallRequirement, Optional[Text], Optional[PackageFinder]) -> Optional[SetupInfo] import pip_shims.shims if not ireq.link: return @@ -658,22 +793,19 @@ class SetupInfo(object): finder = get_finder() vcs_method, uri = split_vcs_method_from_uri(unquote(ireq.link.url_without_fragment)) parsed = urlparse(uri) - url_path = parsed.path - if "@" in url_path: - url_path, _, _ = url_path.rpartition("@") - parsed = parsed._replace(path=url_path) - uri = urlunparse(parsed) + if "file" in parsed.scheme: + url_path = parsed.path + if "@" in url_path: + url_path, _, _ = url_path.rpartition("@") + parsed = parsed._replace(path=url_path) + uri = urlunparse(parsed) path = None if ireq.link.scheme == "file" or uri.startswith("file://"): if "file:/" in uri and "file:///" not in uri: uri = uri.replace("file:/", "file:///") path = pip_shims.shims.url_to_path(uri) - # if pip_shims.shims.is_installable_dir(path) and ireq.editable: - # ireq.source_dir = path kwargs = _prepare_wheel_building_kwargs(ireq) ireq.source_dir = kwargs["src_dir"] - # os.environ["PIP_BUILD_DIR"] = kwargs["build_dir"] - ireq.ensure_has_source_dir(kwargs["build_dir"]) if not ( ireq.editable and pip_shims.shims.is_file_url(ireq.link) @@ -690,17 +822,18 @@ class SetupInfo(object): "The file URL points to a directory not installable: {}" .format(ireq.link) ) - if not ireq.editable: - build_dir = ireq.build_location(kwargs["build_dir"]) - ireq._temp_build_dir.path = kwargs["build_dir"] - else: - build_dir = ireq.build_location(kwargs["src_dir"]) - ireq._temp_build_dir.path = kwargs["build_dir"] + # if not ireq.editable: + build_dir = ireq.build_location(kwargs["build_dir"]) + src_dir = ireq.ensure_has_source_dir(kwargs["src_dir"]) + ireq._temp_build_dir.path = kwargs["build_dir"] + # else: + # build_dir = ireq.build_location(kwargs["src_dir"]) + # ireq._temp_build_dir.path = kwargs["build_dir"] ireq.populate_link(finder, False, False) pip_shims.shims.unpack_url( ireq.link, - build_dir, + src_dir, download_dir, only_download=only_download, session=finder.session, @@ -708,17 +841,17 @@ class SetupInfo(object): progress_bar="off", ) created = cls.create( - build_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs + src_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs ) return created @classmethod def create(cls, base_dir, subdirectory=None, ireq=None, kwargs=None): + # type: (Text, Optional[Text], Optional[InstallRequirement], Optional[Dict[Text, Text]]) -> Optional[SetupInfo] if not base_dir or base_dir is None: return creation_kwargs = {"extra_kwargs": kwargs} - if not isinstance(base_dir, Path): base_dir = Path(base_dir) creation_kwargs["base_dir"] = base_dir.as_posix() diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index a5e781f4..eccf5e66 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -29,14 +29,22 @@ from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri from ..environment import MYPY_RUNNING if MYPY_RUNNING: - from typing import Union, Optional, List, Set, Any, TypeVar, Tuple, Sequence, Dict + from typing import Union, Optional, List, Set, Any, TypeVar, Tuple, Sequence, Dict, Text from attr import _ValidatorType from packaging.requirements import Requirement as PackagingRequirement from pkg_resources import Requirement as PkgResourcesRequirement - from pkg_resources.extern.packaging.markers import Marker as PkgResourcesMarker + from pkg_resources.extern.packaging.markers import ( + Op as PkgResourcesOp, Variable as PkgResourcesVariable, + Value as PkgResourcesValue, Marker as PkgResourcesMarker + ) from pip_shims.shims import Link + from vistir.compat import Path _T = TypeVar("_T") TMarker = Union[Marker, PkgResourcesMarker] + TVariable = TypeVar("TVariable", PkgResourcesVariable, Variable) + TValue = TypeVar("TValue", PkgResourcesValue, Value) + TOp = TypeVar("TOp", PkgResourcesOp, Op) + MarkerTuple = Tuple[TVariable, TOp, TValue] TRequirement = Union[PackagingRequirement, PkgResourcesRequirement] @@ -59,7 +67,7 @@ DIRECT_URL_RE = re.compile(r"{0}\s?@\s?{1}".format(NAME_WITH_EXTRAS, URL)) def filter_none(k, v): - # type: (str, Any) -> bool + # type: (Text, Any) -> bool if v: return True return False @@ -71,7 +79,7 @@ def optional_instance_of(cls): def create_link(link): - # type: (str) -> Link + # type: (Text) -> Link if not isinstance(link, six.string_types): raise TypeError("must provide a string to instantiate a new link") @@ -80,13 +88,13 @@ def create_link(link): def get_url_name(url): - # type: (str) -> str + # type: (Text) -> Text """ Given a url, derive an appropriate name to use in a pipfile. :param str url: A url to derive a string from :returns: The name of the corresponding pipfile entry - :rtype: str + :rtype: Text """ if not isinstance(url, six.string_types): raise TypeError("Expected a string, got {0!r}".format(url)) @@ -94,7 +102,7 @@ def get_url_name(url): def init_requirement(name): - # type: (str) -> TRequirement + # type: (Text) -> TRequirement if not isinstance(name, six.string_types): raise TypeError("must supply a name to generate a requirement") @@ -108,12 +116,11 @@ def init_requirement(name): def extras_to_string(extras): - # type: (Sequence) -> str + # type: (Sequence) -> Text """Turn a list of extras into a string""" if isinstance(extras, six.string_types): if extras.startswith("["): return extras - else: extras = [extras] if not extras: @@ -122,7 +129,7 @@ def extras_to_string(extras): def parse_extras(extras_str): - # type: (str) -> List + # type: (Text) -> List """ Turn a string of extras into a parsed extras list """ @@ -133,7 +140,7 @@ def parse_extras(extras_str): def specs_to_string(specs): - # type: (List[str, Specifier]) -> str + # type: (List[Union[Text, Specifier]]) -> Text """ Turn a list of specifier tuples into a string """ @@ -150,14 +157,14 @@ def specs_to_string(specs): def build_vcs_uri( - vcs, # type: Optional[str] - uri, # type: str - name=None, # type: Optional[str] - ref=None, # type: Optional[str] - subdirectory=None, # type: Optional[str] - extras=None # type: Optional[List[str]] + vcs, # type: Optional[Text] + uri, # type: Text + name=None, # type: Optional[Text] + ref=None, # type: Optional[Text] + subdirectory=None, # type: Optional[Text] + extras=None # type: Optional[List[Text]] ): - # type: (...) -> str + # type: (...) -> Text if extras is None: extras = [] vcs_start = "" @@ -170,52 +177,54 @@ def build_vcs_uri( if name: uri = "{0}#egg={1}".format(uri, name) if extras: - extras = extras_to_string(extras) - uri = "{0}{1}".format(uri, extras) + extras_string = extras_to_string(extras) + uri = "{0}{1}".format(uri, extras_string) if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) return uri def convert_direct_url_to_url(direct_url): - # type: (str) -> str + # type: (Text) -> Text """ Given a direct url as defined by *PEP 508*, convert to a :class:`~pip_shims.shims.Link` compatible URL by moving the name and extras into an **egg_fragment**. :param str direct_url: A pep-508 compliant direct url. :return: A reformatted URL for use with Link objects and :class:`~pip_shims.shims.InstallRequirement` objects. - :rtype: str + :rtype: Text """ direct_match = DIRECT_URL_RE.match(direct_url) if direct_match is None: url_match = URL_RE.match(direct_url) if url_match or is_valid_url(direct_url): - return url_match + return direct_url match_dict = direct_match.groupdict() - url = [match_dict.get(segment) for segment in ("scheme", "host", "path", "pathsep")] - url = "".join([s for s in url if s is not None]) + if not match_dict: + raise ValueError("Failed converting value to normal URL, is it a direct URL? {0!r}".format(direct_url)) + url_segments = [match_dict.get(s) for s in ("scheme", "host", "path", "pathsep")] + url = "".join([s for s in url_segments if s is not None]) new_url = build_vcs_uri( None, url, ref=match_dict.get("ref"), name=match_dict.get("name"), extras=match_dict.get("extras"), - subdir=match_dict.get("subdirectory") + subdirectory=match_dict.get("subdirectory") ) return new_url def convert_url_to_direct_url(url, name=None): - # type: (str, Optional[str]) -> str + # type: (Text, Optional[Text]) -> Text """ Given a :class:`~pip_shims.shims.Link` compatible URL, convert to a direct url as defined by *PEP 508* by extracting the name and extras from the **egg_fragment**. - :param str url: A :class:`~pip_shims.shims.InstallRequirement` compliant URL. - :param Optiona[str] name: A name to use in case the supplied URL doesn't provide one. + :param Text url: A :class:`~pip_shims.shims.InstallRequirement` compliant URL. + :param Optiona[Text] name: A name to use in case the supplied URL doesn't provide one. :return: A pep-508 compliant direct url. - :rtype: str + :rtype: Text :raises ValueError: Raised when the URL can't be parsed or a name can't be found. :raises TypeError: When a non-string input is provided. @@ -228,15 +237,15 @@ def convert_url_to_direct_url(url, name=None): if direct_match: return url url_match = URL_RE.match(url) - if url_match is None: + if url_match is None or not url_match.groupdict(): raise ValueError("Failed parse a valid URL from {0!r}".format(url)) match_dict = url_match.groupdict() - url = [match_dict.get(segment) for segment in ("scheme", "host", "path", "pathsep")] + url_segments = [match_dict.get(s) for s in ("scheme", "host", "path", "pathsep")] name = match_dict.get("name", name) extras = match_dict.get("extras") new_url = "" if extras and not name: - url.append(extras) + url_segments.append(extras) elif extras and name: new_url = "{0}{1}@ ".format(name, extras) else: @@ -248,14 +257,14 @@ def convert_url_to_direct_url(url, name=None): "No name could be parsed from {0!r}".format(url) ) if match_dict.get("ref"): - url.append("@{0}".format(match_dict.get("ref"))) + url_segments.append("@{0}".format(match_dict.get("ref"))) url = "".join([s for s in url if s is not None]) - new_url = "{0}{1}".format(new_url, url) - return new_url + url = "{0}{1}".format(new_url, url) + return url def get_version(pipfile_entry): - # type: (Union[str, Dict[str, bool, List[str]]]) -> str + # type: (Union[Text, Dict[Text, bool, List[Text]]]) -> Text if str(pipfile_entry) == "{}" or is_star(pipfile_entry): return "" @@ -282,13 +291,16 @@ def strip_extras_markers_from_requirement(req): """ if req is None: raise TypeError("Must pass in a valid requirement, received {0!r}".format(req)) - if req.marker is not None: + if getattr(req, "marker", None) is not None: + marker = req.marker # type: TMarker req.marker._markers = _strip_extras_markers(req.marker._markers) + if not req.marker._markers: + req.marker = None return req def _strip_extras_markers(marker): - # type: (TMarker) -> TMarker + # type: (Union[MarkerTuple, List[Union[MarkerTuple, str]]]) -> List[Union[MarkerTuple, str]] if marker is None or not isinstance(marker, (list, tuple)): raise TypeError("Expecting a marker type, received {0!r}".format(marker)) markers_to_remove = [] @@ -310,14 +322,14 @@ def _strip_extras_markers(marker): def get_pyproject(path): - # type: (Union[str, Path]) -> Tuple[List[str], str] + # type: (Union[Text, Path]) -> Tuple[List[Text], Text] """ Given a base path, look for the corresponding ``pyproject.toml`` file and return its build_requires and build_backend. - :param str path: The root path of the project, should be a directory (will be truncated) + :param Text path: The root path of the project, should be a directory (will be truncated) :return: A 2 tuple of build requirements and the build backend - :rtype: Tuple[List[str], str] + :rtype: Tuple[List[Text], Text] """ from vistir.compat import Path @@ -330,8 +342,10 @@ def get_pyproject(path): pp_toml = path.joinpath("pyproject.toml") setup_py = path.joinpath("setup.py") if not pp_toml.exists(): - if setup_py.exists(): + if not setup_py.exists(): return None + requires = ["setuptools>=40.6", "wheel"] + backend = "setuptools.build_meta:__legacy__" else: pyproject_data = {} with io.open(pp_toml.as_posix(), encoding="utf-8") as fh: @@ -339,10 +353,10 @@ def get_pyproject(path): build_system = pyproject_data.get("build-system", None) if build_system is None: if setup_py.exists(): - requires = ["setuptools", "wheel"] - backend = "setuptools.build_meta" + requires = ["setuptools>=40.6", "wheel"] + backend = "setuptools.build_meta:__legacy__" else: - requires = ["setuptools>=38.2.5", "wheel"] + requires = ["setuptools>=40.6", "wheel"] backend = "setuptools.build_meta" build_system = { "requires": requires, @@ -350,13 +364,13 @@ def get_pyproject(path): } pyproject_data["build_system"] = build_system else: - requires = build_system.get("requires") - backend = build_system.get("build-backend", "setuptools.build_meta") - return (requires, backend) + requires = build_system.get("requires", ["setuptools>=40.6", "wheel"]) + backend = build_system.get("build-backend", "setuptools.build_meta:__legacy__") + return (requires, backend) def split_markers_from_line(line): - # type: (str) -> Tuple[str, Optional[str]] + # type: (Text) -> Tuple[Text, Optional[Text]] """Split markers from a dependency""" if not any(line.startswith(uri_prefix) for uri_prefix in SCHEME_LIST): marker_sep = ";" @@ -370,7 +384,7 @@ def split_markers_from_line(line): def split_vcs_method_from_uri(uri): - # type: (str) -> Tuple[Optional[str], str] + # type: (Text) -> Tuple[Optional[Text], Text] """Split a vcs+uri formatted uri into (vcs, uri)""" vcs_start = "{0}+" vcs = first([vcs for vcs in VCS_LIST if uri.startswith(vcs_start.format(vcs))]) @@ -380,14 +394,14 @@ def split_vcs_method_from_uri(uri): def split_ref_from_uri(uri): - # type: (str) -> Tuple[str, Optional[str]] + # type: (Text) -> Tuple[Text, Optional[Text]] """ Given a path or URI, check for a ref and split it from the path if it is present, returning a tuple of the original input and the ref or None. - :param str uri: The path or URI to split + :param Text uri: The path or URI to split :returns: A 2-tuple of the path or URI and the ref - :rtype: Tuple[str, Optional[str]] + :rtype: Tuple[Text, Optional[Text]] """ if not isinstance(uri, six.string_types): raise TypeError("Expected a string, received {0!r}".format(uri)) @@ -780,12 +794,12 @@ def fix_requires_python_marker(requires_python): def normalize_name(pkg): - # type: (str) -> str + # type: (Text) -> Text """Given a package name, return its normalized, non-canonicalized form. - :param str pkg: The name of a package + :param Text pkg: The name of a package :return: A normalized package name - :rtype: str + :rtype: Text """ assert isinstance(pkg, six.string_types) @@ -793,12 +807,12 @@ def normalize_name(pkg): def get_name_variants(pkg): - # type: (str) -> Set[str] + # type: (Text) -> Set[Text] """ Given a packager name, get the variants of its name for both the canonicalized and "safe" forms. - :param str pkg: The package to lookup + :param Text pkg: The package to lookup :returns: A list of names. :rtype: Set """ diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index 65fccf30..a3ddfdde 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -10,17 +10,22 @@ import sys import tomlkit import vistir -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) -from six.moves import Mapping, Sequence, Set, ItemsView +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) # type: ignore # noqa +six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) # type: ignore # noqa +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # type: ignore # noqa +six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) # type: ignore # noqa +from six.moves import Mapping, Sequence, Set, ItemsView # type: ignore # noqa from six.moves.urllib.parse import urlparse, urlsplit, urlunparse import pip_shims.shims from vistir.compat import Path from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir +from .environment import MYPY_RUNNING + +if MYPY_RUNNING: + from typing import Dict, Any, Optional, Union, Tuple, List, Iterable, Generator, Text + VCS_LIST = ("git", "svn", "hg", "bzr") @@ -69,11 +74,12 @@ VCS_SCHEMES = [ def is_installable_dir(path): + # type: (Text) -> bool if pip_shims.shims.is_installable_dir(path): return True - path = Path(path) - pyproject = path.joinpath("pyproject.toml") - if pyproject.exists(): + pyproject_path = os.path.join(path, "pyproject.toml") + if os.path.exists(pyproject_path): + pyproject = Path(pyproject_path) pyproject_toml = tomlkit.loads(pyproject.read_text()) build_system = pyproject_toml.get("build-system", {}).get("build-backend", "") if build_system: @@ -82,7 +88,7 @@ def is_installable_dir(path): def strip_ssh_from_git_uri(uri): - # type: (str) -> str + # type: (Text) -> Text """Return git+ssh:// formatted URI to git+git@ format""" if isinstance(uri, six.string_types): if "git+ssh://" in uri: @@ -99,8 +105,8 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): - # type: (str) -> str - """Cleans VCS uris from pipenv.patched.notpip format""" + # type: (Text) -> Text + """Cleans VCS uris from pip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: @@ -114,6 +120,7 @@ def add_ssh_scheme_to_git_uri(uri): def is_vcs(pipfile_entry): + # type: (Union[Text, Dict[Text, Union[Text, bool, Tuple[Text], List[Text]]]]) -> bool """Determine if dictionary entry from Pipfile is for a vcs dependency.""" if isinstance(pipfile_entry, Mapping): return any(key for key in pipfile_entry.keys() if key in VCS_LIST) @@ -128,6 +135,7 @@ def is_vcs(pipfile_entry): def is_editable(pipfile_entry): + # type: (Union[Text, Dict[Text, Union[Text, bool, Tuple[Text], List[Text]]]]) -> bool if isinstance(pipfile_entry, Mapping): return pipfile_entry.get("editable", False) is True if isinstance(pipfile_entry, six.string_types): @@ -136,6 +144,7 @@ def is_editable(pipfile_entry): def multi_split(s, split): + # type: (Text, Iterable[Text]) -> List[Text] """Splits on multiple given separators.""" for r in split: s = s.replace(r, "|") @@ -143,13 +152,14 @@ def multi_split(s, split): def is_star(val): + # type: (Union[Text, Dict[Text, Union[Text, bool, Tuple[Text], List[Text]]]]) -> bool return (isinstance(val, six.string_types) and val == "*") or ( isinstance(val, Mapping) and val.get("version", "") == "*" ) def convert_entry_to_path(path): - # type: (Dict[str, Any]) -> str + # type: (Dict[Text, Union[Text, bool, Tuple[Text], List[Text]]]) -> Text """Convert a pipfile entry to a string""" if not isinstance(path, Mapping): @@ -167,6 +177,7 @@ def convert_entry_to_path(path): def is_installable_file(path): + # type: (Union[Text, Dict[Text, Union[Text, bool, Tuple[Text], List[Text]]]]) -> bool """Determine if a path can potentially be installed""" from packaging import specifiers @@ -187,7 +198,7 @@ def is_installable_file(path): parsed = urlparse(path) is_local = (not parsed.scheme or parsed.scheme == "file" or (len(parsed.scheme) == 1 and os.name == "nt")) if parsed.scheme and parsed.scheme == "file": - path = vistir.path.url_to_path(path) + path = vistir.compat.fs_decode(vistir.path.url_to_path(path)) normalized_path = vistir.path.normalize_path(path) if is_local and not os.path.exists(normalized_path): return False @@ -220,7 +231,7 @@ def get_dist_metadata(dist): def get_setup_paths(base_path, subdirectory=None): - # type: (str, Optional[str]) -> Dict[str, Optional[str]] + # type: (Text, Optional[Text]) -> Dict[Text, Optional[Text]] if base_path is None: raise TypeError("must provide a path to derive setup paths from") setup_py = os.path.join(base_path, "setup.py") @@ -245,23 +256,24 @@ def get_setup_paths(base_path, subdirectory=None): def prepare_pip_source_args(sources, pip_args=None): + # type: (List[Dict[Text, Union[Text, bool]]], Optional[List[Text]]) -> List[Text] if pip_args is None: pip_args = [] if sources: # Add the source to pip9. - pip_args.extend(["-i", sources[0]["url"]]) + pip_args.extend(["-i", sources[0]["url"]]) # type: ignore # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): - pip_args.extend(["--trusted-host", urlparse(sources[0]["url"]).hostname]) + pip_args.extend(["--trusted-host", urlparse(sources[0]["url"]).hostname]) # type: ignore # Add additional sources as extra indexes. if len(sources) > 1: for source in sources[1:]: - pip_args.extend(["--extra-index-url", source["url"]]) + pip_args.extend(["--extra-index-url", source["url"]]) # type: ignore # Trust the host if it's not verified. if not source.get("verify_ssl", True): pip_args.extend( ["--trusted-host", urlparse(source["url"]).hostname] - ) + ) # type: ignore return pip_args @@ -271,10 +283,11 @@ def _ensure_dir(path): @contextlib.contextmanager -def ensure_setup_py(base_dir): - if not base_dir: - base_dir = create_tracked_tempdir(prefix="requirementslib-setup") - base_dir = Path(base_dir) +def ensure_setup_py(base): + # type: (Text) -> Generator[None, None, None] + if not base: + base = create_tracked_tempdir(prefix="requirementslib-setup") + base_dir = Path(base) if base_dir.exists() and base_dir.name == "setup.py": base_dir = base_dir.parent elif not (base_dir.exists() and base_dir.is_dir()):