From 16c1cc6ebdf78f1c0672a8288ef8ca14609d7111 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 22 Apr 2020 13:58:02 -0400 Subject: [PATCH] Upgrade outdated vendored deps - Upgrade outdated `pip_shims` and `requirementslib` deps ahead of release - Upgrade `importlib-metadata` Signed-off-by: Dan Ryan --- news/4188.vendor.rst | 3 + pipenv/vendor/importlib_metadata/__init__.py | 15 +- pipenv/vendor/importlib_metadata/_compat.py | 17 + .../importlib_metadata/docs/changelog.rst | 11 + .../vendor/importlib_metadata/docs/using.rst | 10 +- .../importlib_metadata/tests/test_main.py | 6 + .../importlib_metadata/tests/test_zip.py | 9 +- pipenv/vendor/pip_shims/__init__.py | 2 +- pipenv/vendor/pip_shims/compat.py | 224 +-- pipenv/vendor/pip_shims/models.py | 60 +- pipenv/vendor/pip_shims/utils.py | 23 + pipenv/vendor/requirementslib/__init__.py | 2 +- .../vendor/requirementslib/models/markers.py | 44 +- .../vendor/requirementslib/models/metadata.py | 1240 +++++++++++++++++ .../vendor/requirementslib/models/pipfile.py | 4 + .../requirementslib/models/setup_info.py | 7 +- pipenv/vendor/requirementslib/models/utils.py | 2 +- pipenv/vendor/vendor.txt | 6 +- 18 files changed, 1568 insertions(+), 117 deletions(-) create mode 100644 pipenv/vendor/requirementslib/models/metadata.py diff --git a/news/4188.vendor.rst b/news/4188.vendor.rst index 9feb035a..e24a45fd 100644 --- a/news/4188.vendor.rst +++ b/news/4188.vendor.rst @@ -6,3 +6,6 @@ Add and update vendored dependencies to accommodate ``safety`` vendoring: - **certifi** ``2019.11.28`` => ``2020.4.5.1`` - **pyparsing** ``2.4.6`` => ``2.4.7`` - **resolvelib** ``0.2.2`` => ``0.3.0`` +- **importlib-metadata** ``1.5.1`` => ``1.6.0`` +- **pip-shims** ``0.5.1`` => ``0.5.2`` +- **requirementslib** ``1.5.5`` => ``1.5.6`` diff --git a/pipenv/vendor/importlib_metadata/__init__.py b/pipenv/vendor/importlib_metadata/__init__.py index 089fca97..95a08ba0 100644 --- a/pipenv/vendor/importlib_metadata/__init__.py +++ b/pipenv/vendor/importlib_metadata/__init__.py @@ -28,6 +28,7 @@ from ._compat import ( MetaPathFinder, email_message_from_string, PyPy_repr, + unique_ordered, ) from importlib import import_module from itertools import starmap @@ -95,6 +96,16 @@ class EntryPoint( attrs = filter(None, (match.group('attr') or '').split('.')) return functools.reduce(getattr, attrs, module) + @property + def module(self): + match = self.pattern.match(self.value) + return match.group('module') + + @property + def attr(self): + match = self.pattern.match(self.value) + return match.group('attr') + @property def extras(self): match = self.pattern.match(self.value) @@ -425,8 +436,8 @@ class FastPath: names = zip_path.root.namelist() self.joinpath = zip_path.joinpath - return ( - posixpath.split(child)[0] + return unique_ordered( + child.split(posixpath.sep, 1)[0] for child in names ) diff --git a/pipenv/vendor/importlib_metadata/_compat.py b/pipenv/vendor/importlib_metadata/_compat.py index 99b4005e..f59b57db 100644 --- a/pipenv/vendor/importlib_metadata/_compat.py +++ b/pipenv/vendor/importlib_metadata/_compat.py @@ -15,9 +15,11 @@ if sys.version_info > (3,): # pragma: nocover NotADirectoryError = builtins.NotADirectoryError PermissionError = builtins.PermissionError map = builtins.map + from itertools import filterfalse else: # pragma: nocover from backports.configparser import ConfigParser from itertools import imap as map # type: ignore + from itertools import ifilterfalse as filterfalse import contextlib2 as contextlib FileNotFoundError = IOError, OSError IsADirectoryError = IOError, OSError @@ -131,3 +133,18 @@ class PyPy_repr: if affected: # pragma: nocover __repr__ = __compat_repr__ del affected + + +# from itertools recipes +def unique_everseen(iterable): # pragma: nocover + "List unique elements, preserving order. Remember all elements ever seen." + seen = set() + seen_add = seen.add + + for element in filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + + +unique_ordered = ( + unique_everseen if sys.version_info < (3, 7) else dict.fromkeys) diff --git a/pipenv/vendor/importlib_metadata/docs/changelog.rst b/pipenv/vendor/importlib_metadata/docs/changelog.rst index 9dbaf96a..d638e38d 100644 --- a/pipenv/vendor/importlib_metadata/docs/changelog.rst +++ b/pipenv/vendor/importlib_metadata/docs/changelog.rst @@ -2,6 +2,17 @@ importlib_metadata NEWS ========================= +v1.6.0 +====== + +* Added ``module`` and ``attr`` attributes to ``EntryPoint`` + +v1.5.2 +====== + +* Fix redundant entries from ``FastPath.zip_children``. + Closes #117. + v1.5.1 ====== diff --git a/pipenv/vendor/importlib_metadata/docs/using.rst b/pipenv/vendor/importlib_metadata/docs/using.rst index 6da2bb1c..d1ca7658 100644 --- a/pipenv/vendor/importlib_metadata/docs/using.rst +++ b/pipenv/vendor/importlib_metadata/docs/using.rst @@ -70,7 +70,9 @@ Entry points The ``entry_points()`` function returns a dictionary of all entry points, keyed by group. Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value:: +a ``.load()`` method to resolve the value. There are also ``.module``, +``.attr``, and ``.extras`` attributes for getting the components of the +``.value`` attribute:: >>> eps = entry_points() >>> list(eps) @@ -79,6 +81,12 @@ a ``.load()`` method to resolve the value:: >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] >>> wheel EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') + >>> wheel.module + 'wheel.cli' + >>> wheel.attr + 'main' + >>> wheel.extras + [] >>> main = wheel.load() >>> main diff --git a/pipenv/vendor/importlib_metadata/tests/test_main.py b/pipenv/vendor/importlib_metadata/tests/test_main.py index 131edcea..876203f6 100644 --- a/pipenv/vendor/importlib_metadata/tests/test_main.py +++ b/pipenv/vendor/importlib_metadata/tests/test_main.py @@ -250,3 +250,9 @@ class TestEntryPoints(unittest.TestCase): """ with self.assertRaises(Exception): json.dumps(self.ep) + + def test_module(self): + assert self.ep.module == 'value' + + def test_attr(self): + assert self.ep.attr is None diff --git a/pipenv/vendor/importlib_metadata/tests/test_zip.py b/pipenv/vendor/importlib_metadata/tests/test_zip.py index 8cbba63a..515f593d 100644 --- a/pipenv/vendor/importlib_metadata/tests/test_zip.py +++ b/pipenv/vendor/importlib_metadata/tests/test_zip.py @@ -1,7 +1,10 @@ import sys import unittest -from .. import distribution, entry_points, files, PackageNotFoundError, version +from .. import ( + distribution, entry_points, files, PackageNotFoundError, + version, distributions, + ) try: from importlib.resources import path @@ -52,6 +55,10 @@ class TestZip(unittest.TestCase): path = str(file.dist.locate_file(file)) assert '.whl/' in path, path + def test_one_distribution(self): + dists = list(distributions(path=sys.path[:1])) + assert len(dists) == 1 + class TestEgg(TestZip): def setUp(self): diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py index 7feca147..3a0188c9 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py @@ -5,7 +5,7 @@ import sys from . import shims -__version__ = "0.5.1" +__version__ = "0.5.2" if "pip_shims" in sys.modules: diff --git a/pipenv/vendor/pip_shims/compat.py b/pipenv/vendor/pip_shims/compat.py index ed99d970..0c125321 100644 --- a/pipenv/vendor/pip_shims/compat.py +++ b/pipenv/vendor/pip_shims/compat.py @@ -19,6 +19,8 @@ from packaging import specifiers from .environment import MYPY_RUNNING from .utils import ( call_function_with_correct_args, + filter_allowed_args, + get_allowed_args, get_method_args, nullcontext, suppress_setattr, @@ -65,6 +67,7 @@ if MYPY_RUNNING: TCommandInstance = TypeVar("TCommandInstance") TCmdDict = Dict[str, Union[Tuple[str, str, str], TCommandInstance]] TInstallRequirement = TypeVar("TInstallRequirement") + TFormatControl = TypeVar("TFormatControl") TShimmedCmdDict = Union[TShim, TCmdDict] TWheelCache = TypeVar("TWheelCache") TPreparer = TypeVar("TPreparer") @@ -352,11 +355,11 @@ def ensure_resolution_dirs(**kwargs): @contextlib.contextmanager def wheel_cache( - wheel_cache_provider, # type: TShimmedFunc - tempdir_manager_provider, # type: TShimmedFunc - cache_dir, # type: str + cache_dir=None, # type: str format_control=None, # type: Any + wheel_cache_provider=None, # type: TShimmedFunc format_control_provider=None, # type: Optional[TShimmedFunc] + tempdir_manager_provider=None, # type: TShimmedFunc ): tempdir_manager_provider = resolve_possible_shim(tempdir_manager_provider) wheel_cache_provider = resolve_possible_shim(wheel_cache_provider) @@ -490,6 +493,7 @@ def get_requirement_set( cache_dir=None, # type: Optional[str] options=None, # type: Optional[Values] install_cmd_provider=None, # type: Optional[TShimmedFunc] + wheel_cache_provider=None, # type: Optional[TShimmedFunc] ): # (...) -> TRequirementSet """ @@ -499,6 +503,8 @@ def get_requirement_set( invalid parameters will be ignored if they are not needed to generate a requirement set on the current pip version. + :param :class:`~pip_shims.models.ShimmedPathCollection` wheel_cache_provider: A + context manager provider which resolves to a `WheelCache` instance :param install_command: A :class:`~pip._internal.commands.install.InstallCommand` instance which is used to generate the finder. :param :class:`~pip_shims.models.ShimmedPathCollection` req_set_provider: A provider @@ -538,6 +544,7 @@ def get_requirement_set( :return: A new requirement set instance :rtype: :class:`~pip._internal.req.req_set.RequirementSet` """ + wheel_cache_provider = resolve_possible_shim(wheel_cache_provider) req_set_provider = resolve_possible_shim(req_set_provider) if install_command is None: install_cmd_provider = resolve_possible_shim(install_cmd_provider) @@ -565,10 +572,13 @@ def get_requirement_set( ) if session is None and "session" in required_args: session = get_session(install_cmd=install_command, options=options) - results["wheel_cache"] = wheel_cache - results["session"] = session - results["wheel_download_dir"] = wheel_download_dir - return call_function_with_correct_args(req_set_provider, **results) + with ExitStack() as stack: + if wheel_cache is None: + wheel_cache = stack.enter_context(wheel_cache_provider(cache_dir=cache_dir)) + results["wheel_cache"] = wheel_cache + results["session"] = session + results["wheel_download_dir"] = wheel_download_dir + return call_function_with_correct_args(req_set_provider, **results) def get_package_finder( @@ -665,11 +675,11 @@ def get_package_finder( ): if target_python and not received_python: tags = target_python.get_tags() - version_impl = set([t[0] for t in tags]) + version_impl = {t[0] for t in tags} # impls = set([v[:2] for v in version_impl]) # impls.remove("py") # impl = next(iter(impls), "py") if not target_python - versions = set([v[2:] for v in version_impl]) + versions = {v[2:] for v in version_impl} build_kwargs.update( { "platform": target_python.platform, @@ -689,6 +699,7 @@ def get_package_finder( def shim_unpack( unpack_fn, # type: TShimmedFunc download_dir, # type str + tempdir_manager_provider, # type: TShimmedFunc ireq=None, # type: Optional[Any] link=None, # type: Optional[Any] location=None, # type Optional[str], @@ -708,6 +719,8 @@ def shim_unpack( :param unpack_fn: A callable or shim referring to the pip implementation :type unpack_fn: Callable :param str download_dir: The directory to download the file to + :param TShimmedFunc tempdir_manager_provider: A callable or shim referring to + `global_tempdir_manager` function from pip or a shimmed no-op context manager :param Optional[:class:`~pip._internal.req.req_install.InstallRequirement`] ireq: an Install Requirement instance, defaults to None :param Optional[:class:`~pip._internal.models.link.Link`] link: A Link instance, @@ -727,31 +740,33 @@ def shim_unpack( """ unpack_fn = resolve_possible_shim(unpack_fn) downloader_provider = resolve_possible_shim(downloader_provider) + tempdir_manager_provider = resolve_possible_shim(tempdir_manager_provider) required_args = inspect.getargs(unpack_fn.__code__).args # type: ignore unpack_kwargs = {"download_dir": download_dir} - if ireq: - if not link and ireq.link: - link = ireq.link - if only_download is None: - only_download = ireq.is_wheel - if hashes is None: - hashes = ireq.hashes(True) - if location is None and getattr(ireq, "source_dir", None): - location = ireq.source_dir - unpack_kwargs.update({"link": link, "location": location}) - if hashes is not None and "hashes" in required_args: - unpack_kwargs["hashes"] = hashes - if "progress_bar" in required_args: - unpack_kwargs["progress_bar"] = progress_bar - if only_download is not None and "only_download" in required_args: - unpack_kwargs["only_download"] = only_download - if session is not None and "session" in required_args: - unpack_kwargs["session"] = session - if "downloader" in required_args and downloader_provider is not None: - assert session is not None - assert progress_bar is not None - unpack_kwargs["downloader"] = downloader_provider(session, progress_bar) - return unpack_fn(**unpack_kwargs) # type: ignore + with tempdir_manager_provider(): + if ireq: + if not link and ireq.link: + link = ireq.link + if only_download is None: + only_download = ireq.is_wheel + if hashes is None: + hashes = ireq.hashes(True) + if location is None and getattr(ireq, "source_dir", None): + location = ireq.source_dir + unpack_kwargs.update({"link": link, "location": location}) + if hashes is not None and "hashes" in required_args: + unpack_kwargs["hashes"] = hashes + if "progress_bar" in required_args: + unpack_kwargs["progress_bar"] = progress_bar + if only_download is not None and "only_download" in required_args: + unpack_kwargs["only_download"] = only_download + if session is not None and "session" in required_args: + unpack_kwargs["session"] = session + if "downloader" in required_args and downloader_provider is not None: + assert session is not None + assert progress_bar is not None + unpack_kwargs["downloader"] = downloader_provider(session, progress_bar) + return unpack_fn(**unpack_kwargs) # type: ignore def _ensure_finder( @@ -860,7 +875,7 @@ def make_preparer( preparer_fn = resolve_possible_shim(preparer_fn) downloader_provider = resolve_possible_shim(downloader_provider) finder_provider = resolve_possible_shim(finder_provider) - required_args = inspect.getargs(preparer_fn.__init__.__code__).args # type: ignore + required_args, required_kwargs = get_allowed_args(preparer_fn) # type: ignore if not req_tracker and not req_tracker_fn and "req_tracker" in required_args: raise TypeError("No requirement tracker and no req tracker generator found!") if "downloader" in required_args and not downloader_provider: @@ -918,6 +933,38 @@ def make_preparer( yield result +@contextlib.contextmanager +def _ensure_wheel_cache( + wheel_cache=None, # type: Optional[Type[TWheelCache]] + wheel_cache_provider=None, # type: Optional[Callable] + format_control=None, # type: Optional[TFormatControl] + format_control_provider=None, # type: Optional[Type[TShimmedFunc]] + options=None, # type: Optional[Values] + cache_dir=None, # type: Optional[str] +): + if wheel_cache is not None: + yield wheel_cache + elif wheel_cache_provider is not None: + with ExitStack() as stack: + cache_dir = getattr(options, "cache_dir", cache_dir) + format_control = getattr( + options, + "format_control", + format_control_provider(None, None), # TFormatControl + ) + wheel_cache = stack.enter_context( + wheel_cache_provider(cache_dir, format_control) + ) + yield wheel_cache + + +def get_ireq_output_path(wheel_cache, ireq): + if getattr(wheel_cache, "get_path_for_link", None): + return wheel_cache.get_path_for_link(ireq.link) + elif getattr(wheel_cache, "cached_wheel", None): + return wheel_cache.cached_wheel(ireq.link, ireq.name).url_without_fragment + + def get_resolver( resolver_fn, # type: TShimmedFunc install_req_provider=None, # type: Optional[TShimmedFunc] @@ -938,6 +985,7 @@ def get_resolver( make_install_req=None, # type: Optional[Callable] install_cmd_provider=None, # type: Optional[TShimmedFunc] install_cmd=None, # type: Optional[TCommandInstance] + use_pep517=True, # type: bool ): # (...) -> TResolver """ @@ -985,6 +1033,7 @@ def get_resolver( to the resolver for actually generating install requirements, if necessary :param Optional[TCommandInstance] install_cmd: The install command used to create the finder, session, and options if needed, defaults to None. + :param bool use_pep517: Whether to use the pep517 build process. :return: A new resolver instance. :rtype: :class:`~pip._internal.legacy_resolve.Resolver` @@ -1049,7 +1098,7 @@ def get_resolver( continue elif val is None and install_cmd is None: raise TypeError( - "Preparer requires a {0} but did not receive one " + "Preparer requires a {} but did not receive one " "and cannot generate one".format(arg) ) elif arg == "session" and val is None: @@ -1059,26 +1108,21 @@ def get_resolver( resolver_kwargs[arg] = val if "make_install_req" in required_args: if make_install_req is None and install_req_provider is not None: + make_install_req_kwargs = { + "isolated": isolated, + "wheel_cache": wheel_cache, + "use_pep517": use_pep517, + } + factory_args, factory_kwargs = filter_allowed_args( + install_req_provider, **make_install_req_kwargs + ) make_install_req = functools.partial( - install_req_provider, - isolated=isolated, - wheel_cache=wheel_cache, - # use_pep517=use_pep517, + install_req_provider, *factory_args, **factory_kwargs ) assert make_install_req is not None resolver_kwargs["make_install_req"] = make_install_req if "isolated" in required_args: resolver_kwargs["isolated"] = isolated - if "wheel_cache" in required_args: - if wheel_cache is None and wheel_cache_provider is not None: - cache_dir = getattr(options, "cache_dir", None) - format_control = getattr( - options, - "format_control", - format_control_provider(None, None), # type: ignore - ) - wheel_cache = wheel_cache_provider(cache_dir, format_control) - resolver_kwargs["wheel_cache"] = wheel_cache resolver_kwargs.update( { "upgrade_strategy": upgrade_strategy, @@ -1090,6 +1134,15 @@ def get_resolver( "preparer": preparer, } ) + if "wheel_cache" in required_args: + with _ensure_wheel_cache( + wheel_cache=wheel_cache, + wheel_cache_provider=wheel_cache_provider, + format_control_provider=format_control_provider, + options=options, + ) as wheel_cache: + resolver_kwargs["wheel_cache"] = wheel_cache + return resolver_fn(**resolver_kwargs) # type: ignore return resolver_fn(**resolver_kwargs) # type: ignore @@ -1255,22 +1308,29 @@ def resolve( # noqa:C901 if session is None: session = get_session(install_cmd=install_command, options=options) if finder is None: - finder = finder_provider(install_command, options=options, session=session) # type: ignore + finder = finder_provider( + install_command, options=options, session=session + ) # type: ignore format_control = getattr(options, "format_control", None) if not format_control: format_control = format_control_provider(None, None) # type: ignore - wheel_cache = wheel_cache_provider( - kwargs["cache_dir"], format_control + wheel_cache = ctx.enter_context( + wheel_cache_provider(kwargs["cache_dir"], format_control) ) # type: ignore ireq.is_direct = True # type: ignore build_location_kwargs = {"build_dir": kwargs["build_dir"], "autodelete": True} call_function_with_correct_args(ireq.build_location, **build_location_kwargs) - # ireq.build_location(kwargs["build_dir"]) # type: ignore if reqset_provider is None: raise TypeError( "cannot resolve without a requirement set provider... failed!" ) - reqset = reqset_provider(install_command, options=options, session=session, wheel_download_dir=wheel_download_dir, **kwargs) # type: ignore + reqset = reqset_provider( + install_command, + options=options, + session=session, + wheel_download_dir=wheel_download_dir, + **kwargs + ) # type: ignore if getattr(reqset, "prepare_files", None): reqset.add_requirement(ireq) results = reqset.prepare_files(finder) @@ -1293,7 +1353,6 @@ def resolve( # noqa:C901 "use_user_site": use_user_site, "require_hashes": require_hashes, } - # with req_tracker_provider() as req_tracker: if isinstance(req_tracker_provider, (types.FunctionType, functools.partial)): preparer_args["req_tracker"] = ctx.enter_context(req_tracker_provider()) resolver_keys = [ @@ -1338,7 +1397,7 @@ def resolve( # noqa:C901 return results -def build_wheel( +def build_wheel( # noqa:C901 req=None, # type: Optional[TInstallRequirement] reqset=None, # type: Optional[Union[TReqSet, Iterable[TInstallRequirement]]] output_dir=None, # type: Optional[str] @@ -1368,8 +1427,9 @@ def build_wheel( build_many_provider=None, # type: Optional[TShimmedFunc] install_command_provider=None, # type: Optional[TShimmedFunc] finder_provider=None, # type: Optional[TShimmedFunc] + reqset_provider=None, # type: Optional[TShimmedFunc] ): - # type: (...) -> Optional[Union[str, Tuple[List[TInstallRequirement], List[TInstallRequirement]]]] + # type: (...) -> Generator[Union[str, Tuple[List[TInstallRequirement], ...]], None, None] """ Build a wheel or a set of wheels @@ -1421,19 +1481,21 @@ def build_wheel( :param Optional[TShimmedFunc] install_command_provider: A shim for providing new install command instances :param TShimmedFunc finder_provider: A provider to package finder instances + :param TShimmedFunc reqset_provider: A provider for requirement set generation :return: A tuple of successful and failed install requirements or else a path to a wheel :rtype: Optional[Union[str, Tuple[List[TInstallRequirement], List[TInstallRequirement]]]] """ wheel_cache_provider = resolve_possible_shim(wheel_cache_provider) - preparer = resolve_possible_shim(preparer) + preparer_provider = resolve_possible_shim(preparer_provider) wheel_builder_provider = resolve_possible_shim(wheel_builder_provider) build_one_provider = resolve_possible_shim(build_one_provider) build_one_inside_env_provider = resolve_possible_shim(build_one_inside_env_provider) build_many_provider = resolve_possible_shim(build_many_provider) install_cmd_provider = resolve_possible_shim(install_command_provider) format_control_provider = resolve_possible_shim(format_control_provider) - finder_provider = resolve_possible_shim(finder_provider) + finder_provider = resolve_possible_shim(finder_provider) or get_package_finder + reqset_provider = resolve_possible_shim(reqset_provider) global_options = [] if global_options is None else global_options build_options = [] if build_options is None else build_options options = None @@ -1447,25 +1509,31 @@ def build_wheel( } if not req and not reqset: raise TypeError("Must provide either a requirement or requirement set to build") - if wheel_cache is None and (reqset is not None or output_dir is None): - if install_command is None: - assert isinstance(install_cmd_provider, (type, functools.partial)) - install_command = install_cmd_provider() - kwargs, options = populate_options(install_command, options, **kwarg_map) - format_control = getattr(options, "format_control", None) - if not format_control: - format_control = format_control_provider(None, None) # type: ignore - wheel_cache = wheel_cache_provider(options.cache_dir, format_control) - if req and not reqset and not output_dir: - output_dir = wheel_cache.get_path_for_link(req.link) - if not reqset and build_one_provider: - yield build_one_provider(req, output_dir, build_options, global_options) - elif build_many_provider: - yield build_many_provider( - reqset, wheel_cache, build_options, global_options, check_binary_allowed - ) - else: - with ExitStack() as ctx: + with ExitStack() as ctx: + kwargs = kwarg_map.copy() + if wheel_cache is None and (reqset is not None or output_dir is None): + if install_command is None: + assert isinstance(install_cmd_provider, (type, functools.partial)) + install_command = install_cmd_provider() + kwargs, options = populate_options(install_command, options, **kwarg_map) + format_control = getattr(options, "format_control", None) + if not format_control: + format_control = format_control_provider(None, None) # type: ignore + wheel_cache = ctx.enter_context( + wheel_cache_provider(options.cache_dir, format_control) + ) + if req and not reqset and not output_dir: + output_dir = get_ireq_output_path(wheel_cache, req) + if not reqset and build_one_provider: + yield build_one_provider(req, output_dir, build_options, global_options) + elif build_many_provider: + yield build_many_provider( + reqset, wheel_cache, build_options, global_options, check_binary_allowed + ) + else: + builder_args, builder_kwargs = get_allowed_args(wheel_builder_provider) + if "requirement_set" in builder_args and not reqset: + reqset = reqset_provider() if session is None and finder is None: session = get_session(install_cmd=install_command, options=options) finder = finder_provider( @@ -1503,7 +1571,7 @@ def build_wheel( ) if req and not reqset: if not output_dir: - output_dir = wheel_cache.get_path_for_link(req.link) + output_dir = get_ireq_output_path(wheel_cache, req) if use_pep517 is not None: req.use_pep517 = use_pep517 yield builder._build_one(req, output_dir) diff --git a/pipenv/vendor/pip_shims/models.py b/pipenv/vendor/pip_shims/models.py index d641c6dd..79871f5b 100644 --- a/pipenv/vendor/pip_shims/models.py +++ b/pipenv/vendor/pip_shims/models.py @@ -104,6 +104,9 @@ PIP_VERSION_SET = { "19.2.3", "19.3", "19.3.1", + "20.0", + "20.0.1", + "20.0.2", } @@ -140,9 +143,9 @@ class PipVersion(Sequence): parsed_version = self._parse() if base_import_path is None: if parsed_version >= parse_version("10.0.0"): - base_import_path = "{0}._internal".format(BASE_IMPORT_PATH) + base_import_path = "{}._internal".format(BASE_IMPORT_PATH) else: - base_import_path = "{0}".format(BASE_IMPORT_PATH) + base_import_path = "{}".format(BASE_IMPORT_PATH) self.base_import_path = base_import_path self.parsed_version = parsed_version @@ -175,13 +178,12 @@ class PipVersion(Sequence): def __str__(self): # type: () -> str - return "{0!s}".format(self.parsed_version) + return "{!s}".format(self.parsed_version) def __repr__(self): # type: () -> str return ( - "" + "" ).format( self.version, self.base_import_path, @@ -253,17 +255,17 @@ class PipVersionRange(Sequence): def __str__(self): # type: () -> str - return "{0!s} -> {1!s}".format(self._versions[0], self._versions[-1]) + return "{!s} -> {!s}".format(self._versions[0], self._versions[-1]) @property def base_import_paths(self): # type: () -> Set[str] - return set([version.base_import_path for version in self._versions]) + return {version.base_import_path for version in self._versions} @property def vendor_import_paths(self): # type: () -> Set[str] - return set([version.vendor_import_path for version in self._versions]) + return {version.vendor_import_path for version in self._versions} def is_valid(self): # type: () -> bool @@ -430,7 +432,7 @@ class ShimmedPath(object): if not self.is_class: return provided if not inspect.isclass(provided): - raise TypeError("Provided argument is not a class: {0!r}".format(provided)) + raise TypeError("Provided argument is not a class: {!r}".format(provided)) methods = self._parse_provides_dict( self.provided_methods, prepend_arg_to_callables="self" ) @@ -554,9 +556,7 @@ class ShimmedPath(object): imported, result = self._shim_parent(imported, attribute_name) if result is not None: result = self._ensure_functions(result) - full_import_path = "{0}.{1}".format( - self.calculated_module_path, attribute_name - ) + full_import_path = "{}.{}".format(self.calculated_module_path, attribute_name) self._imported = imported assert isinstance(result, types.ModuleType) self._provided = result @@ -921,13 +921,6 @@ unpack_url = ShimmedPathCollection("unpack_url", ImportTypes.FUNCTION) unpack_url.create_path("download.unpack_url", "7.0.0", "19.3.9") unpack_url.create_path("operations.prepare.unpack_url", "20.0", "9999") -shim_unpack = ShimmedPathCollection("shim_unpack", ImportTypes.FUNCTION) -shim_unpack.set_default( - functools.partial( - compat.shim_unpack, unpack_fn=unpack_url, downloader_provider=Downloader - ) -) - is_installable_dir = ShimmedPathCollection("is_installable_dir", ImportTypes.FUNCTION) is_installable_dir.create_path("utils.misc.is_installable_dir", "10.0.0", "9999") is_installable_dir.create_path("utils.is_installable_dir", "7.0.0", "9.0.3") @@ -1036,6 +1029,16 @@ global_tempdir_manager.create_path( "utils.temp_dir.global_tempdir_manager", "7.0.0", "9999" ) +shim_unpack = ShimmedPathCollection("shim_unpack", ImportTypes.FUNCTION) +shim_unpack.set_default( + functools.partial( + compat.shim_unpack, + unpack_fn=unpack_url, + downloader_provider=Downloader, + tempdir_manager_provider=global_tempdir_manager, + ) +) + get_requirement_tracker = ShimmedPathCollection( "get_requirement_tracker", ImportTypes.CONTEXTMANAGER ) @@ -1128,6 +1131,17 @@ DEV_PKGS.create_path("commands.freeze.DEV_PKGS", "9.0.0", "9999") DEV_PKGS.set_default({"setuptools", "pip", "distribute", "wheel"}) +wheel_cache = ShimmedPathCollection("wheel_cache", ImportTypes.FUNCTION) +wheel_cache.set_default( + functools.partial( + compat.wheel_cache, + wheel_cache_provider=WheelCache, + tempdir_manager_provider=global_tempdir_manager, + format_control_provider=FormatControl, + ) +) + + get_package_finder = ShimmedPathCollection("get_package_finder", ImportTypes.FUNCTION) get_package_finder.set_default( functools.partial( @@ -1158,7 +1172,7 @@ get_resolver.set_default( install_cmd_provider=InstallCommand, resolver_fn=Resolver, install_req_provider=install_req_from_req_string, - wheel_cache_provider=WheelCache, + wheel_cache_provider=wheel_cache, format_control_provider=FormatControl, ) ) @@ -1170,6 +1184,7 @@ get_requirement_set.set_default( compat.get_requirement_set, install_cmd_provider=InstallCommand, req_set_provider=RequirementSet, + wheel_cache_provider=wheel_cache, ) ) @@ -1182,7 +1197,7 @@ resolve.set_default( reqset_provider=get_requirement_set, finder_provider=get_package_finder, resolver_provider=get_resolver, - wheel_cache_provider=WheelCache, + wheel_cache_provider=wheel_cache, format_control_provider=FormatControl, make_preparer_provider=make_preparer, req_tracker_provider=get_requirement_tracker, @@ -1196,12 +1211,13 @@ build_wheel.set_default( functools.partial( compat.build_wheel, install_command_provider=InstallCommand, - wheel_cache_provider=WheelCache, + wheel_cache_provider=wheel_cache, wheel_builder_provider=WheelBuilder, build_one_provider=build_one, build_one_inside_env_provider=build_one_inside_env, build_many_provider=build, preparer_provider=make_preparer, format_control_provider=FormatControl, + reqset_provider=get_requirement_set, ) ) diff --git a/pipenv/vendor/pip_shims/utils.py b/pipenv/vendor/pip_shims/utils.py index 76661ecc..162b4a20 100644 --- a/pipenv/vendor/pip_shims/utils.py +++ b/pipenv/vendor/pip_shims/utils.py @@ -440,3 +440,26 @@ def call_function_with_correct_args(fn, **provided_kwargs): continue kwargs[arg] = provided_kwargs[arg] return fn(*args, **kwargs) + + +def filter_allowed_args(fn, **provided_kwargs): + # type: (Callable, Dict[str, Any]) -> Tuple[List[Any], Dict[str, Any]] + """ + Given a function and a kwarg mapping, return only those kwargs used in the function. + + :param Callable fn: A function to inspect + :param Dict[str, Any] kwargs: A mapping of kwargs to filter + :return: A new, filtered kwarg mapping + :rtype: Tuple[List[Any], Dict[str, Any]] + """ + args = [] + kwargs = {} + func_args, func_kwargs = get_allowed_args(fn) + for arg in func_args: + if arg in provided_kwargs: + args.append(provided_kwargs[arg]) + for arg in func_kwargs: + if arg not in provided_kwargs: + continue + kwargs[arg] = provided_kwargs[arg] + return args, kwargs diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 488da849..f78192c9 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -10,7 +10,7 @@ from .models.lockfile import Lockfile from .models.pipfile import Pipfile from .models.requirements import Requirement -__version__ = "1.5.4" +__version__ = "1.5.6" logger = logging.getLogger(__name__) diff --git a/pipenv/vendor/requirementslib/models/markers.py b/pipenv/vendor/requirementslib/models/markers.py index 8dda0495..84637642 100644 --- a/pipenv/vendor/requirementslib/models/markers.py +++ b/pipenv/vendor/requirementslib/models/markers.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import itertools import operator +import re import attr import distlib.markers @@ -196,15 +197,19 @@ def _get_specs(specset): return sorted(result, key=operator.itemgetter(1)) +# TODO: Rename this to something meaningful def _group_by_op(specs): # type: (Union[Set[Specifier], SpecifierSet]) -> Iterator specs = [_get_specs(x) for x in list(specs)] - flattened = [(op, version) for spec in specs for op, version in spec] + flattened = [ + ((op, len(version) > 2), version) for spec in specs for op, version in spec + ] specs = sorted(flattened) grouping = itertools.groupby(specs, key=operator.itemgetter(0)) return grouping +# TODO: rename this to something meaningful def normalize_specifier_set(specs): # type: (Union[str, SpecifierSet]) -> Optional[Set[Specifier]] """Given a specifier set, a string, or an iterable, normalize the specifiers @@ -235,6 +240,8 @@ def normalize_specifier_set(specs): return normalize_specifier_set(SpecifierSet(",".join(spec_list))) +# TODO: Check if this is used by anything public otherwise make it private +# And rename it to something meaningful def get_sorted_version_string(version_set): # type: (Set[AnyStr]) -> AnyStr version_list = sorted( @@ -244,6 +251,9 @@ def get_sorted_version_string(version_set): return version +# TODO: Rename this to something meaningful +# TODO: Add a deprecation decorator and deprecate this -- i'm sure it's used +# in other libraries @lru_cache(maxsize=1024) def cleanup_pyspecs(specs, joiner="or"): specs = normalize_specifier_set(specs) @@ -275,7 +285,8 @@ def cleanup_pyspecs(specs, joiner="or"): "==": lambda x: "in" if len(x) > 1 else "==", } translation_keys = list(translation_map.keys()) - for op, versions in _group_by_op(tuple(specs)): + for op_and_version_type, versions in _group_by_op(tuple(specs)): + op = op_and_version_type[0] versions = [version[1] for version in versions] versions = sorted(dedup(versions)) op_key = next(iter(k for k in translation_keys if op in k), None) @@ -284,10 +295,11 @@ def cleanup_pyspecs(specs, joiner="or"): version_value = translation_map[op_key][joiner](versions) if op in op_translations: op = op_translations[op](versions) - results[op] = version_value - return sorted([(k, v) for k, v in results.items()], key=operator.itemgetter(1)) + results[(op, op_and_version_type[1])] = version_value + return sorted([(k[0], v) for k, v in results.items()], key=operator.itemgetter(1)) +# TODO: Rename this to something meaningful @lru_cache(maxsize=1024) def fix_version_tuple(version_tuple): # type: (Tuple[AnyStr, AnyStr]) -> Tuple[AnyStr, AnyStr] @@ -302,6 +314,7 @@ def fix_version_tuple(version_tuple): return (op, version) +# TODO: Rename this to something meaningful, deprecate it (See prior function) @lru_cache(maxsize=128) def get_versions(specset, group_by_operator=True): # type: (Union[Set[Specifier], SpecifierSet], bool) -> List[Tuple[STRING_TYPE, STRING_TYPE]] @@ -590,6 +603,7 @@ def get_specset(marker_list): return specifiers +# TODO: Refactor this (reduce complexity) def parse_marker_dict(marker_dict): op = marker_dict["op"] lhs = marker_dict["lhs"] @@ -658,9 +672,16 @@ def parse_marker_dict(marker_dict): return specset, finalized_marker +def _contains_micro_version(version_string): + return re.search("\d+\.\d+\.\d+", version_string) is not None + + def format_pyversion(parts): op, val = parts - return "python_version {0} '{1}'".format(op, val) + version_marker = ( + "python_full_version" if _contains_micro_version(val) else "python_version" + ) + return "{0} {1} '{2}'".format(version_marker, op, val) def normalize_marker_str(marker): @@ -699,3 +720,16 @@ def marker_from_specifier(spec): marker_segments.append(format_pyversion(marker_segment)) marker_str = " and ".join(marker_segments).replace('"', "'") return Marker(marker_str) + + +def merge_markers(m1, m2): + # type: (Marker, Marker) -> Optional[Marker] + if not all((m1, m2)): + return next(iter(v for v in (m1, m2) if v), None) + m1 = _ensure_marker(m1) + m2 = _ensure_marker(m2) + _markers = [] # type: List[Marker] + for marker in (m1, m2): + _markers.append(str(marker)) + marker_str = " and ".join([normalize_marker_str(m) for m in _markers if m]) + return _ensure_marker(normalize_marker_str(marker_str)) diff --git a/pipenv/vendor/requirementslib/models/metadata.py b/pipenv/vendor/requirementslib/models/metadata.py new file mode 100644 index 00000000..912b1b77 --- /dev/null +++ b/pipenv/vendor/requirementslib/models/metadata.py @@ -0,0 +1,1240 @@ +# -*- coding=utf-8 -*- +import datetime +import functools +import io +import json +import logging +import operator +import os +import zipfile +from collections import defaultdict + +import attr +import dateutil.parser +import distlib.metadata +import distlib.wheel +import packaging.version +import requests +import six +import vistir +from packaging.markers import Marker +from packaging.requirements import Requirement as PackagingRequirement +from packaging.specifiers import Specifier, SpecifierSet +from packaging.tags import Tag + +from ..environment import MYPY_RUNNING +from .markers import ( + get_contained_extras, + get_contained_pyversions, + get_without_extra, + get_without_pyversion, + marker_from_specifier, + merge_markers, + normalize_specifier_set, +) +from .requirements import Requirement +from .utils import filter_dict, get_pinned_version, is_pinned_requirement + +# fmt: off +from six.moves import Sequence # type: ignore # isort:skip +from six.moves import reduce # type: ignore # isort:skip +# fmt: on # isort:skip + + +ch = logging.StreamHandler() +formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s") +ch.setFormatter(formatter) +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +if MYPY_RUNNING: + from typing import ( + Any, + Callable, + Dict, + Generator, + Generic, + Iterator, + List, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, + ) + from attr import Attribute # noqa + from .setup_info import SetupInfo + + TAttrsClass = TypeVar("TAttrsClass") + AttrsClass = Generic[TAttrsClass] + TDigestDict = Dict[str, str] + TProjectUrls = Dict[str, str] + TReleaseUrlDict = Dict[str, Union[bool, int, str, TDigestDict]] + TReleasesList = List[TReleaseUrlDict] + TReleasesDict = Dict[str, TReleasesList] + TDownloads = Dict[str, int] + TPackageInfo = Dict[str, Optional[Union[str, List[str], TDownloads, TProjectUrls]]] + TPackage = Dict[str, Union[TPackageInfo, int, TReleasesDict, TReleasesList]] + + +VALID_ALGORITHMS = { + "sha1": 40, + "sha3_224": 56, + "sha512": 128, + "blake2b": 128, + "sha256": 64, + "sha384": 96, + "blake2s": 64, + "sha3_256": 64, + "sha3_512": 128, + "md5": 32, + "sha3_384": 96, + "sha224": 56, +} # type: Dict[str, int] + +PACKAGE_TYPES = { + "sdist", + "bdist_wheel", + "bdist_egg", + "bdist_dumb", + "bdist_wininst", + "bdist_rpm", + "bdist_msi", + "bdist_dmg", +} + + +class PackageEncoder(json.JSONEncoder): + def default(self, obj): # noqa:E0202 # noqa:W0221 + if isinstance(obj, datetime.datetime): + return obj.isoformat() + elif isinstance(obj, PackagingRequirement): + return obj.__dict__ + elif isinstance(obj, set): + return tuple(obj) + elif isinstance(obj, (Specifier, SpecifierSet, Marker)): + return str(obj) + else: + return json.JSONEncoder.default(self, obj) + + +def validate_extras(inst, attrib, value): + # type: ("Dependency", Attribute, Tuple[str, ...]) -> None + duplicates = [k for k in value if value.count(k) > 1] + if duplicates: + raise ValueError("Found duplicate keys: {0}".format(", ".join(duplicates))) + return None + + +def validate_digest(inst, attrib, value): + # type: ("Digest", Attribute, str) -> None + expected_length = VALID_ALGORITHMS[inst.algorithm.lower()] + if len(value) != expected_length: + raise ValueError( + "Expected a digest of length {0!s}, got one of length {1!s}".format( + expected_length, len(value) + ) + ) + return None + + +def get_local_wheel_metadata(wheel_file): + # type: (str) -> Optional[distlib.metadata.Metadata] + parsed_metadata = None + with io.open(wheel_file, "rb") as fh: + with zipfile.ZipFile(fh, mode="r", compression=zipfile.ZIP_DEFLATED) as zf: + metadata = None + for fn in zf.namelist(): + if os.path.basename(fn) == "METADATA": + metadata = fn + break + if metadata is None: + raise RuntimeError("No metadata found in wheel: {0}".format(wheel_file)) + with zf.open(metadata, "r") as metadata_fh: + parsed_metadata = distlib.metadata.Metadata(fileobj=metadata_fh) + return parsed_metadata + + +def get_remote_sdist_metadata(line): + # type: (str) -> SetupInfo + req = Requirement.from_line(line) + try: + _ = req.run_requires() + except SystemExit: + raise RuntimeError("Failed to compute metadata for dependency {0}".format(line)) + else: + return req.line_instance.setup_info + + +def get_remote_wheel_metadata(whl_file): + # type: (str) -> Optional[distlib.metadata.Metadata] + parsed_metadata = None + data = io.BytesIO() + with vistir.contextmanagers.open_file(whl_file) as fp: + for chunk in iter(lambda: fp.read(8096), b""): + data.write(chunk) + with zipfile.ZipFile(data, mode="r", compression=zipfile.ZIP_DEFLATED) as zf: + metadata = None + for fn in zf.namelist(): + if os.path.basename(fn) == "METADATA": + metadata = fn + break + if metadata is None: + raise RuntimeError("No metadata found in wheel: {0}".format(whl_file)) + with zf.open(metadata, "r") as metadata_fh: + parsed_metadata = distlib.metadata.Metadata(fileobj=metadata_fh) + return parsed_metadata + + +def create_specifierset(spec=None): + # type: (Optional[str]) -> SpecifierSet + if isinstance(spec, SpecifierSet): + return spec + elif isinstance(spec, (set, list, tuple)): + spec = " and ".join(spec) + if spec is None: + spec = "" + return SpecifierSet(spec) + + +@attr.s(frozen=True, eq=True) +class ExtrasCollection(object): + #: The name of the extras collection (e.g. 'security') + name = attr.ib(type=str) + #: The dependency the collection belongs to + parent = attr.ib(type="Dependency") + #: The members of the collection + dependencies = attr.ib(factory=set) # type: Set["Dependency"] + + def add_dependency(self, dependency): + # type: ("Dependency") -> "ExtrasCollection" + if not isinstance(dependency, Dependency): + raise TypeError( + "Expected a Dependency instance, received {0!r}".format(dependency) + ) + dependencies = self.dependencies.copy() + dependencies.add(dependency) + return attr.evolve(self, dependencies=dependencies) + + +@attr.s(frozen=True, eq=True) +class Dependency(object): + #: The name of the dependency + name = attr.ib(type=str) + #: A requirement instance + requirement = attr.ib(type=PackagingRequirement, eq=False) + #: The specifier defined in the dependency definition + specifier = attr.ib(type=SpecifierSet, converter=create_specifierset, eq=False) + #: Any extras this dependency declares + extras = attr.ib(factory=tuple, validator=validate_extras) # type: Tuple[str, ...] + #: The name of the extra meta-dependency this one came from (e.g. 'security') + from_extras = attr.ib(default=None, eq=False) # type: Optional[str] + #: The declared specifier set of allowable python versions for this dependency + python_version = attr.ib( + default="", type=SpecifierSet, converter=create_specifierset, eq=False + ) + #: The parent of this dependency (i.e. where it came from) + parent = attr.ib(default=None) # type: Optional[Dependency] + #: The markers for this dependency + markers = attr.ib(default=None, eq=False) # type: Optional[Marker] + _specset_str = attr.ib(default="", type=str) + _python_version_str = attr.ib(default="", type=str) + _marker_str = attr.ib(default="", type=str) + + def __str__(self): + # type: () -> str + return str(self.requirement) + + def as_line(self): + # type: () -> str + line_str = "{0}".format(self.name) + if self.extras: + line_str = "{0}[{1}]".format(line_str, ",".join(self.extras)) + if self.specifier: + line_str = "{0}{1!s}".format(line_str, self.specifier) + py_version_part = "" + if self.python_version: + specifiers = normalize_specifier_set(self.python_version) + markers = [] + if specifiers is not None: + markers = [marker_from_specifier(str(s)) for s in specifiers] + py_version_part = reduce(merge_markers, markers) + if self.markers: + line_str = "{0}; {1}".format(line_str, str(self.markers)) + if py_version_part: + line_str = "{0} and {1}".format(line_str, py_version_part) + elif py_version_part and not self.markers: + line_str = "{0}; {1}".format(line_str, py_version_part) + return line_str + + def pin(self): + # type: () -> "Package" + base_package = get_package(self.name) + sorted_releases = sorted( + base_package.releases.non_yanked_releases, + key=operator.attrgetter("parsed_version"), + reverse=True, + ) + version = next( + iter(self.specifier.filter((r.version for r in sorted_releases))), None + ) + if not version: + version = next( + iter( + self.specifier.filter( + (r.version for r in sorted_releases), prereleases=True + ) + ), + None, + ) + if not version: + raise RuntimeError( + "Failed to resolve {0} ({1!s})".format(self.name, self.specifier) + ) + match = get_package_version(self.name, str(version)) + return match + + @classmethod + def from_requirement(cls, req, parent=None): + # type: (PackagingRequirement, Optional["Dependency"]) -> "Dependency" + from_extras, marker, python_version = None, None, None + specset_str, py_version_str, marker_str = "", "", "" + if req.marker: + marker = Marker(str(req.marker)) + from_extras = next(iter(list(get_contained_extras(marker))), None) + python_version = get_contained_pyversions(marker) + marker = get_without_extra(get_without_pyversion(marker)) + if not str(marker) or not marker or not marker._markers: + marker = None + req.marker = marker + if marker is not None: + marker_str = str(marker) + if req.specifier: + specset_str = str(req.specifier) + if python_version: + py_version_str = str(python_version) + return cls( + name=req.name, + specifier=req.specifier, + extras=tuple(sorted(set(req.extras))) if req.extras is not None else req.extras, + requirement=req, + from_extras=from_extras, + python_version=python_version, + markers=marker, + parent=parent, + specset_str=specset_str, + python_version_str=py_version_str, + marker_str=marker_str, + ) + + @classmethod + def from_info(cls, info): + # type: ("PackageInfo") -> "Dependency" + marker_str = "" + specset_str, py_version_str = "", "" + if info.requires_python: + # XXX: Some markers are improperly formatted -- we already handle most cases + # XXX: but learned about new broken formats, such as + # XXX: python_version in "2.6 2.7 3.2 3.3" (note the lack of commas) + # XXX: as a marker on a dependency of a library called 'pickleshare' + # XXX: Some packages also have invalid markers with stray characters, + # XXX: such as 'algoliasearch' + try: + marker = marker_from_specifier(info.requires_python) + except Exception: + marker_str = "" + else: + if not marker or not marker._markers: + marker_str = "" + else: + marker_str = "{0!s}".format(marker) + req_str = "{0}=={1}".format(info.name, info.version) + if marker_str: + req_str = "{0}; {1}".format(req_str, marker_str) + req = PackagingRequirement(req_str) + requires_python_str = ( + info.requires_python if info.requires_python is not None else "" + ) + if req.specifier: + specset_str = str(req.specifier) + if requires_python_str: + py_version_str = requires_python_str + return cls( + name=info.name, + specifier=req.specifier, + extras=tuple(sorted(set(req.extras))) if req.extras is not None else req.extras, + requirement=req, + from_extras=None, + python_version=SpecifierSet(requires_python_str), + markers=None, + parent=None, + specset_str=specset_str, + python_version_str=py_version_str, + marker_str=marker_str, + ) + + @classmethod + def from_str(cls, depstr, parent=None): + # type: (str, Optional["Dependency"]) -> "Dependency" + try: + req = PackagingRequirement(depstr) + except Exception: + raise + return cls.from_requirement(req, parent=parent) + + def add_parent(self, parent): + # type: ("Dependency") -> "Dependency" + return attr.evolve(self, parent=parent) + + +@attr.s(frozen=True, eq=True) +class Digest(object): + #: The algorithm declared for the digest, e.g. 'sha256' + algorithm = attr.ib( + type=str, validator=attr.validators.in_(VALID_ALGORITHMS.keys()), eq=True + ) + #: The digest value + value = attr.ib(type=str, validator=validate_digest, eq=True) + + def __str__(self): + # type: () -> str + return "{0}:{1}".format(self.algorithm, self.value) + + @classmethod + def create(cls, algorithm, value): + # type: (str, str) -> "Digest" + return cls(algorithm=algorithm, value=value) + + @classmethod + def collection_from_dict(cls, digest_dict): + # type: (TDigestDict) -> List["Digest"] + return [cls.create(k, v) for k, v in digest_dict.items()] + + +# XXX: This is necessary because attrs converters can only be functions, not classmethods +def create_digest_collection(digest_dict): + # type: (TDigestDict) -> List["Digest"] + return Digest.collection_from_dict(digest_dict) + + +def instance_check_converter(expected_type=None, converter=None): + # type: (Optional[Type], Optional[Callable]) -> Callable + def _converter(val): + if expected_type is not None and isinstance(val, expected_type): + return val + return converter(val) + + return _converter + + +@attr.s(frozen=True, eq=True) +class ParsedTag(object): + #: The marker string corresponding to the tag + marker_string = attr.ib(default=None) # type: Optional[str] + #: The python version represented by the tag + python_version = attr.ib(default=None) # type: Optional[str] + #: The platform represented by the tag + platform_system = attr.ib(default=None) # type: Optional[str] + #: the ABI represented by the tag + abi = attr.ib(default=None) # type: Optional[str] + + +def parse_tag(tag): + # type: (Tag) -> ParsedTag + """ + Parse a :class:`~packaging.tags.Tag` instance + + :param :class:`~packaging.tags.Tag` tag: A tag to parse + :return: A parsed tag with combined markers, supported platform and python version + :rtype: :class:`~ParsedTag` + """ + platform_system = None + python_version = None + version = None + marker_str = "" + if tag.platform.startswith("macos"): + platform_system = "Darwin" + elif tag.platform.startswith("manylinux") or tag.platform.startswith("linux"): + platform_system = "Linux" + elif tag.platform.startswith("win32"): + platform_system = "Windows" + if platform_system: + marker_str = 'platform_system == "{}"'.format(platform_system) + if tag.interpreter: + version = tag.interpreter[2:] + py_version_str = "" + if len(version) == 1: + py_version_str = ">={}.0,<{}".format(version, str(int(version) + 1)) + elif len(version) > 1 and len(version) <= 3: + # reverse the existing version so we can add 1 to the first element + # and re-reverse, generating the new version, e.g. [3, 2, 8] => + # [8, 2, 3] => [9, 2, 3] => [3, 2, 9] + next_version_list = list(reversed(version[:])) + next_version_list[0] = str(int(next_version_list[0]) + 1) + next_version = ".".join(list(reversed(next_version_list))) + version = ".".join(version) + py_version_str = ">={},<{}".format(version, next_version) + else: + py_version_str = "{0}".format(version) + python_version = marker_from_specifier(py_version_str) + if python_version: + if marker_str: + marker_str = "{0} and {1!s}".format(marker_str, python_version) + else: + marker_str = str(python_version) + return ParsedTag( + marker_string=marker_str, + python_version=version, + platform_system=platform_system, + abi=tag.abi, + ) + + +@attr.s(frozen=True, eq=True) +class ReleaseUrl(object): + #: The MD5 digest of the given release + md5_digest = attr.ib(type=Digest) + #: The package type of the url + packagetype = attr.ib(type=str, validator=attr.validators.in_(PACKAGE_TYPES)) + #: The upload timestamp from the package + upload_time = attr.ib( + type=datetime.datetime, + converter=instance_check_converter(datetime.datetime, dateutil.parser.parse), # type: ignore + ) + #: The ISO8601 formatted upload timestamp of the package + upload_time_iso_8601 = attr.ib( + type=datetime.datetime, + converter=instance_check_converter(datetime.datetime, dateutil.parser.parse), # type: ignore + ) + #: The size in bytes of the package + size = attr.ib(type=int) + #: The URL of the package + url = attr.ib(type=str) + #: The digests of the package + digests = attr.ib( + converter=instance_check_converter(list, create_digest_collection) # type: ignore + ) # type: List[Digest] + #: The name of the package + name = attr.ib(type=str, default=None) + #: The available comments of the given upload + comment_text = attr.ib(type=str, default="") + #: The number of downloads (deprecated) + downloads = attr.ib(type=int, default=-1) + #: The filename of the current upload + filename = attr.ib(type=str, default="") + #: Whether the upload has a signature + has_sig = attr.ib(type=bool, default=False) + #: The python_version attribute of the upload (e.g. 'source', 'py27', etc) + python_version = attr.ib(type=str, default="source") + #: The 'requires_python' restriction on the package + requires_python = attr.ib(type=str, default=None) + #: A list of valid aprsed tags from the upload + tags = attr.ib(factory=list) # type: List[ParsedTag] + + @property + def is_wheel(self): + # type: () -> bool + return os.path.splitext(self.filename)[-1].lower() == ".whl" + + @property + def is_sdist(self): + # type: () -> bool + return self.python_version == "source" + + @property + def markers(self): + # type: () -> Optional[str] + # TODO: Compare dependencies in parent and add markers for python version + # TODO: Compare dependencies in parent and add markers for platform + # XXX: We can't use wheel-based markers until we do it via derived markers by + # XXX: comparing in the parent (i.e. 'Release' instance or so) and merging + # XXX: down to the common / minimal set of markers otherwise we wind up + # XXX: with an unmanageable set and combinatorial explosion + # if self.is_wheel: + # return self.get_markers_from_wheel() + if self.requires_python: + return marker_from_specifier(self.requires_python) + return None + + @property + def pep508_url(self): + # type: () -> str + markers = self.markers + req_str = "{0} @ {1}#egg={0}".format(self.name, self.url) + if markers: + req_str = "{0}; {1}".format(req_str, markers) + return req_str + + def get_markers_from_wheel(self): + # type: () -> str + supported_platforms = [] # type: List[str] + supported_pyversions = [] + supported_abis = [] + markers = [] + for parsed_tag in self.tags: + if parsed_tag.marker_string: + markers.append(Marker(parsed_tag.marker_string)) + if parsed_tag.python_version: + supported_pyversions.append(parsed_tag.python_version) + if parsed_tag.abi: + supported_abis.append(parsed_tag.abi) + if not (markers or supported_platforms): + return "" + if ( + all(pyversion in supported_pyversions for pyversion in ["2", "3"]) + and not supported_platforms + ): + marker_line = "" + else: + marker_line = " or ".join(["{}".format(str(marker)) for marker in markers]) + return marker_line + + def get_dependencies(self): + # type: () -> Tuple["ReleaseUrl", Dict[str, Union[List[str], str]]] + results = {"requires_python": None} + requires_dist = [] # type: List[str] + if self.is_wheel: + metadata = get_remote_wheel_metadata(self.url) + if metadata is not None: + requires_dist = metadata.run_requires + if not self.requires_python: + results["requires_python"] = metadata._legacy.get("Requires-Python") + else: + try: + metadata = get_remote_sdist_metadata(self.pep508_url) + except Exception: + requires_dist = [] + else: + requires_dist = [str(v) for v in metadata.requires.values()] + results["requires_dist"] = requires_dist + requires_python = getattr(self, "requires_python", results["requires_python"]) + return attr.evolve(self, requires_python=requires_python), results + + @property + def sha256(self): + # type: () -> str + return next( + iter(digest for digest in self.digests if digest.algorithm == "sha256") + ).value + + @classmethod + def create(cls, release_dict, name=None): + # type: (TReleaseUrlDict, Optional[str]) -> "ReleaseUrl" + valid_digest_keys = set("{0}_digest".format(k) for k in VALID_ALGORITHMS.keys()) + digest_keys = set(release_dict.keys()) & valid_digest_keys + creation_kwargs = {} # type: Dict[str, Union[bool, int, str, Digest, TDigestDict]] + creation_kwargs = { + k: v for k, v in release_dict.items() if k not in digest_keys + } + if name is not None: + creation_kwargs["name"] = name + for k in digest_keys: + digest = release_dict[k] + if not isinstance(digest, six.string_types): + raise TypeError("Digests must be strings, got {!r}".format(digest)) + creation_kwargs[k] = Digest.create(k.replace("_digest", ""), digest) + release_url = cls(**filter_dict(creation_kwargs)) # type: ignore + if release_url.is_wheel: + supported_tags = [ + parse_tag(Tag(*tag)) for tag in distlib.wheel.Wheel(release_url.url).tags + ] + release_url = attr.evolve(release_url, tags=supported_tags) + return release_url + + +def create_release_urls_from_list(urls, name=None): + # type: (Union[TReleasesList, List[ReleaseUrl]], Optional[str]) -> List[ReleaseUrl] + url_list = [] + for release_dict in urls: + if isinstance(release_dict, ReleaseUrl): + if name and not release_dict.name: + release_dict = attr.evolve(release_dict, name=name) + url_list.append(release_dict) + continue + url_list.append(ReleaseUrl.create(release_dict, name=name)) + return url_list + + +@attr.s(frozen=True, eq=True) +class ReleaseUrlCollection(Sequence): + #: A list of release URLs + urls = attr.ib(converter=create_release_urls_from_list) + #: the name of the package + name = attr.ib(default=None) # type: Optional[str] + + @classmethod + def create(cls, urls, name=None): + # type: (TReleasesList, Optional[str]) -> "ReleaseUrlCollection" + return cls(urls=urls, name=name) + + @property + def wheels(self): + # type: () -> Iterator[ReleaseUrl] + for url in self.urls: + if not url.is_wheel: + continue + yield url + + @property + def sdists(self): + # type: () -> Iterator[ReleaseUrl] + for url in self.urls: + if not url.is_sdist: + continue + yield url + + def __iter__(self): + # type: () -> Iterator[ReleaseUrl] + return iter(self.urls) + + def __getitem__(self, key): + # type: (int) -> ReleaseUrl + return self.urls.__getitem__(key) + + def __len__(self): + # type: () -> int + return len(self.urls) + + @property + def latest(self): + # type: () -> Optional[ReleaseUrl] + if not self.urls: + return None + return next( + iter(sorted(self.urls, key=operator.attrgetter("upload_time"), reverse=True)) + ) + + @property + def latest_timestamp(self): + # type: () -> Optional[datetime.datetime] + latest = self.latest + if latest is not None: + return latest.upload_time + return None + + def find_package_type(self, type_): + # type: (str) -> Optional[ReleaseUrl] + """ + Given a package type (e.g. sdist, bdist_wheel), find the matching release + + :param str type_: A package type from :const:`~PACKAGE_TYPES` + :return: The package from this collection matching that type, if available + :rtype: Optional[ReleaseUrl] + """ + if type_ not in PACKAGE_TYPES: + raise ValueError( + "Invalid package type: {0}. Expected one of {1}".format( + type_, " ".join(PACKAGE_TYPES) + ) + ) + return next(iter(url for url in self.urls if url.packagetype == type_), None) + + +def convert_release_urls_to_collection(urls=None, name=None): + # type: (Optional[TReleasesList], Optional[str]) -> ReleaseUrlCollection + if urls is None: + urls = [] + urls = create_release_urls_from_list(urls, name=name) + return ReleaseUrlCollection.create(urls, name=name) + + +@attr.s(frozen=True) +class Release(Sequence): + #: The version of the release + version = attr.ib(type=str) + #: The URL collection for the release + urls = attr.ib( + converter=instance_check_converter( # type: ignore + ReleaseUrlCollection, convert_release_urls_to_collection + ), + type=ReleaseUrlCollection, + ) + #: the name of the package + name = attr.ib(default=None) # type: Optional[str] + + def __iter__(self): + # type: () -> Iterator[ReleaseUrlCollection] + return iter(self.urls) + + def __getitem__(self, key): + return self.urls[key] + + def __len__(self): + # type: () -> int + return len(self.urls) + + @property + def yanked(self): + # type: () -> bool + if not self.urls: + return True + return False + + @property + def parsed_version(self): + # type: () -> packaging.version._BaseVersion + return packaging.version.parse(self.version) + + @property + def wheels(self): + # type: () -> Iterator[ReleaseUrl] + return self.urls.wheels + + @property + def sdists(self): + # type: () -> Iterator[ReleaseUrl] + return self.urls.sdists + + @property + def latest(self): + # type: () -> ReleaseUrl + return self.urls.latest + + @property + def latest_timestamp(self): + # type: () -> datetime.datetime + return self.urls.latest_timestamp + + def to_lockfile(self): + # type: () -> Dict[str, Union[List[str], str]] + return { + "hashes": [str(url.sha256) for url in self.urls if url.sha256 is not None], + "version": "=={0}".format(self.version), + } + + +def get_release(version, urls, name=None): + # type: (str, TReleasesList, Optional[str]) -> Release + release_kwargs = {"version": version, "name": name} + if not isinstance(urls, ReleaseUrlCollection): + release_kwargs["urls"] = convert_release_urls_to_collection(urls, name=name) + else: + release_kwargs["urls"] = urls + return Release(**release_kwargs) # type: ignore + + +def get_releases_from_package(releases, name=None): + # type: (TReleasesDict, Optional[str]) -> List[Release] + release_list = [] + for version, urls in releases.items(): + release_list.append(get_release(version, urls, name=name)) + return release_list + + +@attr.s(frozen=True) +class ReleaseCollection(object): + releases = attr.ib( + factory=list, converter=instance_check_converter(list, get_releases_from_package), # type: ignore + ) # type: List[Release] + + def __iter__(self): + # type: () -> Iterator[Release] + return iter(self.releases) + + def __getitem__(self, key): + # type: (str) -> Release + result = next(iter(r for r in self.releases if r.version == key), None) + if result is None: + raise KeyError(key) + return result + + def __len__(self): + # type: () -> int + return len(self.releases) + + def get_latest_lockfile(self): + # type: () -> Dict[str, Union[str, List[str]]] + return self.latest.to_lockfile() + + def wheels(self): + # type: () -> Iterator[ReleaseUrl] + for release in self.sort_releases(): + for wheel in release.wheels: + yield wheel + + def sdists(self): + # type: () -> Iterator[ReleaseUrl] + for release in self.sort_releases(): + for sdist in release.sdists: + yield sdist + + @property + def non_yanked_releases(self): + # type: () -> List[Release] + return list(r for r in self.releases if not r.yanked) + + def sort_releases(self): + # type: () -> List[Release] + return sorted( + self.non_yanked_releases, + key=operator.attrgetter("latest_timestamp"), + reverse=True, + ) + + @property + def latest(self): + # type: () -> Optional[Release] + return next(iter(r for r in self.sort_releases() if not r.yanked)) + + @classmethod + def load(cls, releases, name=None): + # type: (Union[TReleasesDict, List[Release]], Optional[str]) -> "ReleaseCollection" + if not isinstance(releases, list): + releases = get_releases_from_package(releases, name=name) + return cls(releases) + + +def convert_releases_to_collection(releases, name=None): + # type: (TReleasesDict, Optional[str]) -> ReleaseCollection + return ReleaseCollection.load(releases, name=name) + + +def split_keywords(value): + # type: (Union[str, List]) -> List[str] + if value and isinstance(value, six.string_types): + return value.split(",") + elif isinstance(value, list): + return value + return [] + + +def create_dependencies( + requires_dist, # type: Optional[List[Dependency]] + parent=None, # type: Optional[Dependency] +): + # type: (...) -> Optional[Set[Dependency]] + if requires_dist is None: + return None + dependencies = set() + for req in requires_dist: + if not isinstance(req, Dependency): + dependencies.add(Dependency.from_str(req, parent=parent)) + else: + dependencies.add(req) + return dependencies + + +@attr.s(frozen=True) +class PackageInfo(object): + name = attr.ib(type=str) + version = attr.ib(type=str) + package_url = attr.ib(type=str) + summary = attr.ib(type=str, default=None) # type: Optional[str] + author = attr.ib(type=str, default=None) # type: Optional[str] + keywords = attr.ib(factory=list, converter=split_keywords) # type: List[str] + description = attr.ib(type=str, default="") + download_url = attr.ib(type=str, default="") + home_page = attr.ib(type=str, default="") + license = attr.ib(type=str, default="") + maintainer = attr.ib(type=str, default="") + maintainer_email = attr.ib(type=str, default="") + downloads = attr.ib(factory=dict) # type: Dict[str, int] + docs_url = attr.ib(default=None) # type: Optional[str] + platform = attr.ib(type=str, default="") + project_url = attr.ib(type=str, default="") + project_urls = attr.ib(factory=dict) # type: Dict[str, str] + requires_python = attr.ib(default=None) # type: Optional[str] + requires_dist = attr.ib(factory=list) # type: List[Dependency] + release_url = attr.ib(default=None) # type: Optional[str] + description_content_type = attr.ib(type=str, default="text/md") + bugtrack_url = attr.ib(default=None) # type: str + classifiers = attr.ib(factory=list) # type: List[str] + author_email = attr.ib(default=None) # type: Optional[str] + markers = attr.ib(default=None) # type: Optional[str] + dependencies = attr.ib(default=None) # type: Tuple[Dependency] + + @classmethod + def from_json(cls, info_json): + # type: (TPackageInfo) -> "PackageInfo" + return cls(**filter_dict(info_json)) # type: ignore + + def to_dependency(self): + # type: () -> Dependency + return Dependency.from_info(self) + + def create_dependencies(self, force=False): + # type: (bool) -> "PackageInfo" + """ + Create values for **self.dependencies**. + + :param bool force: Sets **self.dependencies** to an empty tuple if it would be + None, defaults to False. + :return: An updated instance of the current object with **self.dependencies** + updated accordingly. + :rtype: :class:`PackageInfo` + """ + if not self.dependencies and not self.requires_dist: + if force: + return attr.evolve(self, dependencies=tuple()) + return self + self_dependency = self.to_dependency() + deps = set() + self_dependencies = tuple() if not self.dependencies else self.dependencies + for dep in self_dependencies: + if dep is None: + continue + new_dep = dep.add_parent(self_dependency) + deps.add(new_dep) + created_deps = create_dependencies(self.requires_dist, parent=self_dependency) + if created_deps is not None: + for dep in created_deps: + if dep is None: + continue + deps.add(dep) + return attr.evolve(self, dependencies=tuple(sorted(deps))) + + +def convert_package_info(info_json): + # type: (Union[TPackageInfo, PackageInfo]) -> PackageInfo + if isinstance(info_json, PackageInfo): + return info_json + return PackageInfo.from_json(info_json) + + +def add_markers_to_dep(d, marker_str): + # type: (str, Union[str, Marker]) -> str + req = PackagingRequirement(d) + existing_marker = getattr(req, "marker", None) + if isinstance(marker_str, Marker): + marker_str = str(marker_str) + if existing_marker is not None: + marker_str = str(merge_markers(existing_marker, marker_str)) + if marker_str: + marker_str = marker_str.replace("'", '"') + req.marker = Marker(marker_str) + return str(req) + + +@attr.s +class Package(object): + info = attr.ib(type=PackageInfo, converter=convert_package_info) + last_serial = attr.ib(type=int) + releases = attr.ib( + type=ReleaseCollection, + converter=instance_check_converter( # type: ignore + ReleaseCollection, convert_releases_to_collection + ), + ) + # XXX: Note: sometimes releases have no urls at the top level (e.g. pyrouge) + urls = attr.ib( + type=ReleaseUrlCollection, + converter=instance_check_converter( # type: ignore + ReleaseUrlCollection, convert_release_urls_to_collection + ), + ) + + @urls.default + def _get_urls_collection(self): + return functools.partial(convert_release_urls_to_collection, urls=[], name=self.name) + + @property + def name(self): + # type: () -> str + return self.info.name + + @property + def version(self): + # type: () -> str + return self.info.version + + @property + def requirement(self): + # type: () -> PackagingRequirement + return self.info.to_dependency().requirement + + @property + def latest_sdist(self): + # type: () -> ReleaseUrl + return next(iter(self.urls.sdists)) + + @property + def latest_wheels(self): + # type: () -> Iterator[ReleaseUrl] + for wheel in self.urls.wheels: + yield wheel + + @property + def dependencies(self): + # type: () -> List[Dependency] + if self.info.dependencies is None and list(self.urls): + rval = self.get_dependencies() + return rval.dependencies + return list(self.info.dependencies) + + def get_dependencies(self): + # type: () -> "Package" + urls = [] # type: List[ReleaseUrl] + deps = set() # type: Set[str] + info = self.info + if info.dependencies is None: + for url in self.urls: + try: + url, dep_dict = url.get_dependencies() + except (RuntimeError, TypeError): + # This happens if we are parsing `setup.py` and we fail + if url.is_sdist: + continue + else: + raise + markers = url.markers + dep_list = dep_dict.get("requires_dist", []) + for dep in dep_list: + # XXX: We need to parse these as requirements and "and" the markers + # XXX: together because they may contain "extra" markers which we + # XXX: will need to parse and remove + deps.add(add_markers_to_dep(dep, markers)) + urls.append(url) + if None in deps: + deps.remove(None) + info = attr.evolve( + self.info, requires_dist=tuple(sorted(deps)) + ).create_dependencies(force=True) + return attr.evolve(self, info=info, urls=urls) + + @classmethod + def from_json(cls, package_json): + # type: (Dict[str, Any]) -> "Package" + info = convert_package_info(package_json["info"]).create_dependencies() + releases = convert_releases_to_collection( + package_json["releases"], name=info.name + ) + urls = convert_release_urls_to_collection(package_json["urls"], name=info.name) + return cls( + info=info, + releases=releases, + urls=urls, + last_serial=package_json["last_serial"], + ) + + def pin_dependencies(self, include_extras=None): + # type: (Optional[List[str]]) -> Tuple[List["Package"], Dict[str, List[SpecifierSet]]] + deps = [] + if include_extras: + include_extras = list(sorted(set(include_extras))) + else: + include_extras = [] + constraints = defaultdict(list) + for dep in self.dependencies: + if dep.from_extras and dep.from_extras not in include_extras: + continue + if dep.specifier: + constraints[dep.name].append(dep.specifier) + try: + pinned = dep.pin() + except requests.exceptions.HTTPError: + continue + deps.append(pinned) + return deps, constraints + + def get_latest_lockfile(self): + # type: () -> Dict[str, Dict[str, Union[List[str], str]]] + lockfile = {} + constraints = {dep.name: dep.specifier for dep in self.dependencies} + deps, _ = self.pin_dependencies() + for dep in deps: + dep = dep.get_dependencies() + for sub_dep in dep.dependencies: + if sub_dep.name not in constraints: + logger.info( + "Adding {0} (from {1}) {2!s}".format( + sub_dep.name, dep.name, sub_dep.specifier + ) + ) + constraints[sub_dep.name] = sub_dep.specifier + else: + existing = "{0} (from {1}): {2!s} + ".format( + sub_dep.name, dep.name, constraints[sub_dep.name] + ) + new_specifier = sub_dep.specifier + merged = constraints[sub_dep.name] & new_specifier + logger.info( + "Updating: {0}{1!s} = {2!s}".format( + existing, new_specifier, merged + ) + ) + constraints[sub_dep.name] = merged + + lockfile.update({dep.info.name: dep.releases.get_latest_lockfile()}) + for sub_dep_name, specset in constraints.items(): + try: + sub_dep_pkg = get_package(sub_dep_name) + except requests.exceptions.HTTPError: + continue + logger.info("Getting package: {0} ({1!s})".format(sub_dep, specset)) + sorted_releases = list( + sorted( + sub_dep_pkg.releases, + key=operator.attrgetter("parsed_version"), + reverse=True, + ) + ) + try: + version = next(iter(specset.filter((r.version for r in sorted_releases)))) + except StopIteration: + logger.info("No version of {0} matches specifier: {1}".format(sub_dep, specset)) + logger.info( + "Available versions: {0}".format( + " ".join([r.version for r in sorted_releases]) + ) + ) + raise + sub_dep_instance = get_package_version(sub_dep_name, version=str(version)) + if sub_dep_instance is None: + continue + lockfile.update( + { + sub_dep_instance.info.name: sub_dep_instance.releases.get_latest_lockfile() + } + ) + # lockfile.update(dep.get_latest_lockfile()) + lockfile.update({self.info.name: self.releases.get_latest_lockfile()}) + return lockfile + + def as_dict(self): + # type: () -> Dict[str, Any] + return json.loads(self.serialize()) + + def serialize(self): + # type: () -> str + return json.dumps(attr.asdict(self), cls=PackageEncoder, indent=4) + + +def get_package(name): + # type: (str) -> Package + url = "https://pypi.org/pypi/{}/json".format(name) + with requests.get(url) as r: + r.raise_for_status() + result = r.json() + package = Package.from_json(result) + return package + + +def get_package_version(name, version): + # type: (str, str) -> Package + url = "https://pypi.org/pypi/{0}/{1}/json".format(name, version) + with requests.get(url) as r: + r.raise_for_status() + result = r.json() + package = Package.from_json(result) + return package + + +def get_package_from_requirement(req): + # type: (PackagingRequirement) -> Tuple[Package, Set[str]] + versions = set() + if is_pinned_requirement(req): + version = get_pinned_version(req) + versions.add(version) + pkg = get_package_version(req.name, version) + else: + pkg = get_package(req.name) + sorted_releases = list( + sorted(pkg.releases, key=operator.attrgetter("parsed_version"), reverse=True) + ) + versions = set(req.specifier.filter((r.version for r in sorted_releases))) + version = next(iter(req.specifier.filter((r.version for r in sorted_releases)))) + if pkg.version not in versions: + pkg = get_package_version(pkg.name, version) + return pkg, versions diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index db1f5dde..9c0aea4e 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -164,6 +164,10 @@ class Pipfile(object): # type: () -> Union[plette.pipfiles.Pipfile, PipfileLoader] return self.projectfile.model + @property + def root(self): + return self.path.parent + @property def extended_keys(self): return [ diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 81b4b715..22fe51a1 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -679,7 +679,7 @@ AST_COMPARATORS = dict( (ast.And, operator.and_), (ast.Or, operator.or_), (ast.Not, operator.not_), - (ast.In, operator.contains), + (ast.In, lambda a, b: operator.contains(b, a)), ) ) @@ -746,6 +746,9 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no unparsed = item.s elif isinstance(item, ast.Subscript): unparsed = unparse(item.value) + if not initial_mapping: + if isinstance(item.slice, ast.Index): + unparsed = unparsed[unparse(item.slice.value)] elif any(isinstance(item, k) for k in AST_BINOP_MAP.keys()): unparsed = AST_BINOP_MAP[type(item)] elif isinstance(item, ast.Num): @@ -789,7 +792,7 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no elif isinstance(item, constant): unparsed = item.value elif isinstance(item, ast.Compare): - if isinstance(item.left, ast.Attribute): + if isinstance(item.left, ast.Attribute) or isinstance(item.left, ast.Str): import importlib left = unparse(item.left) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 9b6beb64..6c3b7de8 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -723,7 +723,7 @@ def get_pinned_version(ireq): except AttributeError: raise TypeError("Expected InstallRequirement, not {}".format(type(ireq).__name__)) - if ireq.editable: + if getattr(ireq, "editable", False): raise ValueError("InstallRequirement is editable") if not specifier: raise ValueError("InstallRequirement has no version specification") diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 15c12d51..1a06b43b 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -26,7 +26,7 @@ requests==2.23.0 idna==2.9 urllib3==1.25.9 certifi==2020.4.5.1 -requirementslib==1.5.5 +requirementslib==1.5.6 attrs==19.3.0 distlib==0.3.0 packaging==20.3 @@ -39,7 +39,7 @@ semver==2.9.0 toml==0.10.0 cached-property==1.5.1 vistir==0.5.0 -pip-shims==0.5.1 +pip-shims==0.5.2 contextlib2==0.6.0.post1 funcsigs==1.0.2 enum34==1.1.6 @@ -50,7 +50,7 @@ resolvelib==0.3.0 backports.functools_lru_cache==1.5 pep517==0.8.2 zipp==0.6.0 - importlib_metadata==1.5.1 + importlib_metadata==1.6.0 importlib-resources==1.4.0 more-itertools==5.0.0 git+https://github.com/sarugaku/passa.git@master#egg=passa