update to pip 23.3

This commit is contained in:
Matt Davis
2023-10-15 22:02:30 -04:00
committed by Oz Tiram
parent 74f3843687
commit 94ea6e842f
78 changed files with 2903 additions and 685 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
from typing import List, Optional
__version__ = "23.2.1"
__version__ = "23.3"
def main(args: Optional[List[str]] = None) -> int:
-1
View File
@@ -1,6 +1,5 @@
from typing import List, Optional
import pipenv.patched.pip._internal.utils.inject_securetransport # noqa
from pipenv.patched.pip._internal.utils import _log
# init_logging() must be called before any call to logging.getLogger()
+2 -4
View File
@@ -78,12 +78,10 @@ class Cache:
if can_not_cache:
return []
candidates = []
path = self.get_path_for_link(link)
if os.path.isdir(path):
for candidate in os.listdir(path):
candidates.append((candidate, path))
return candidates
return [(candidate, path) for candidate in os.listdir(path)]
return []
def get_path_for_link(self, link: Link) -> str:
"""Return a directory to store cached items in for link."""
@@ -71,8 +71,9 @@ def autocomplete() -> None:
for opt in subcommand.parser.option_list_all:
if opt.help != optparse.SUPPRESS_HELP:
for opt_str in opt._long_opts + opt._short_opts:
options.append((opt_str, opt.nargs))
options += [
(opt_str, opt.nargs) for opt_str in opt._long_opts + opt._short_opts
]
# filter out previously specified options from available options
prev_opts = [x.split("=")[0] for x in cwords[1 : cword - 1]]
@@ -181,7 +181,7 @@ class Command(CommandContextMixIn):
assert isinstance(status, int)
return status
except DiagnosticPipError as exc:
logger.error("[present-rich] %s", exc)
logger.error("%s", exc, extra={"rich": True})
logger.debug("Exception information:", exc_info=True)
return ERROR
@@ -92,10 +92,10 @@ def check_dist_restriction(options: Values, check_target: bool = False) -> None:
)
if check_target:
if dist_restriction_set and not options.target_dir:
if not options.dry_run and dist_restriction_set and not options.target_dir:
raise CommandError(
"Can not use any platform or abi specific options unless "
"installing via '--target'"
"installing via '--target' or using '--dry-run'"
)
@@ -670,7 +670,10 @@ def prefer_binary() -> Option:
dest="prefer_binary",
action="store_true",
default=False,
help="Prefer older binary packages over newer source packages.",
help=(
"Prefer binary packages over source packages, even if the "
"source packages are newer."
),
)
@@ -823,7 +826,7 @@ def _handle_config_settings(
) -> None:
key, sep, val = value.partition("=")
if sep != "=":
parser.error(f"Arguments to {opt_str} must be of the form KEY=VAL") # noqa
parser.error(f"Arguments to {opt_str} must be of the form KEY=VAL")
dest = getattr(parser.values, option.dest)
if dest is None:
dest = {}
@@ -918,13 +921,13 @@ def _handle_merge_hash(
algo, digest = value.split(":", 1)
except ValueError:
parser.error(
"Arguments to {} must be a hash name " # noqa
"Arguments to {} must be a hash name "
"followed by a value, like --hash=sha256:"
"abcde...".format(opt_str)
)
if algo not in STRONG_HASHES:
parser.error(
"Allowed hash algorithms for {} are {}.".format( # noqa
"Allowed hash algorithms for {} are {}.".format(
opt_str, ", ".join(STRONG_HASHES)
)
)
+2 -2
View File
@@ -229,7 +229,7 @@ class ConfigOptionParser(CustomOptionParser):
val = strtobool(val)
except ValueError:
self.error(
"{} is not a valid value for {} option, " # noqa
"{} is not a valid value for {} option, "
"please specify a boolean value like yes/no, "
"true/false or 1/0 instead.".format(val, key)
)
@@ -240,7 +240,7 @@ class ConfigOptionParser(CustomOptionParser):
val = int(val)
if not isinstance(val, int) or val < 0:
self.error(
"{} is not a valid value for {} option, " # noqa
"{} is not a valid value for {} option, "
"please instead specify either a non-negative integer "
"or a boolean value like yes/no or false/true "
"which is equivalent to 1/0.".format(val, key)
@@ -58,12 +58,9 @@ def _create_truststore_ssl_context() -> Optional["SSLContext"]:
return None
try:
import truststore
except ImportError:
raise CommandError(
"To use the truststore feature, 'truststore' must be installed into "
"pip's current environment."
)
from pipenv.patched.pip._vendor import truststore
except ImportError as e:
raise CommandError(f"The truststore feature is unavailable: {e}")
return truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
@@ -123,7 +120,7 @@ class SessionCommandMixin(CommandContextMixIn):
ssl_context = None
session = PipSession(
cache=os.path.join(cache_dir, "http") if cache_dir else None,
cache=os.path.join(cache_dir, "http-v2") if cache_dir else None,
retries=retries if retries is not None else options.retries,
trusted_hosts=options.trusted_hosts,
index_urls=self._get_index_urls(options),
@@ -268,7 +265,7 @@ class RequirementCommand(IndexGroupCommand):
if "legacy-resolver" in options.deprecated_features_enabled:
return "legacy"
return "2020-resolver"
return "resolvelib"
@classmethod
def make_requirement_preparer(
@@ -290,7 +287,7 @@ class RequirementCommand(IndexGroupCommand):
legacy_resolver = False
resolver_variant = cls.determine_resolver_variant(options)
if resolver_variant == "2020-resolver":
if resolver_variant == "resolvelib":
lazy_wheel = "fast-deps" in options.features_enabled
if lazy_wheel:
logger.warning(
@@ -352,7 +349,7 @@ class RequirementCommand(IndexGroupCommand):
# The long import name and duplicated invocation is needed to convince
# Mypy into correctly typechecking. Otherwise it would complain the
# "Resolver" class being redefined.
if resolver_variant == "2020-resolver":
if resolver_variant == "resolvelib":
import pipenv.patched.pip._internal.resolution.resolvelib.resolver
return pipenv.patched.pip._internal.resolution.resolvelib.resolver.Resolver(
+18 -15
View File
@@ -3,10 +3,10 @@ import textwrap
from optparse import Values
from typing import Any, List
import pipenv.patched.pip._internal.utils.filesystem as filesystem
from pipenv.patched.pip._internal.cli.base_command import Command
from pipenv.patched.pip._internal.cli.status_codes import ERROR, SUCCESS
from pipenv.patched.pip._internal.exceptions import CommandError, PipError
from pipenv.patched.pip._internal.utils import filesystem
from pipenv.patched.pip._internal.utils.logging import getLogger
logger = getLogger(__name__)
@@ -93,24 +93,30 @@ class CacheCommand(Command):
num_http_files = len(self._find_http_files(options))
num_packages = len(self._find_wheels(options, "*"))
http_cache_location = self._cache_dir(options, "http")
http_cache_location = self._cache_dir(options, "http-v2")
old_http_cache_location = self._cache_dir(options, "http")
wheels_cache_location = self._cache_dir(options, "wheels")
http_cache_size = filesystem.format_directory_size(http_cache_location)
http_cache_size = filesystem.format_size(
filesystem.directory_size(http_cache_location)
+ filesystem.directory_size(old_http_cache_location)
)
wheels_cache_size = filesystem.format_directory_size(wheels_cache_location)
message = (
textwrap.dedent(
"""
Package index page cache location: {http_cache_location}
Package index page cache location (pip v23.3+): {http_cache_location}
Package index page cache location (older pips): {old_http_cache_location}
Package index page cache size: {http_cache_size}
Number of HTTP files: {num_http_files}
Locally built wheels location: {wheels_cache_location}
Locally built wheels size: {wheels_cache_size}
Number of locally built wheels: {package_count}
"""
""" # noqa: E501
)
.format(
http_cache_location=http_cache_location,
old_http_cache_location=old_http_cache_location,
http_cache_size=http_cache_size,
num_http_files=num_http_files,
wheels_cache_location=wheels_cache_location,
@@ -151,14 +157,8 @@ class CacheCommand(Command):
logger.info("\n".join(sorted(results)))
def format_for_abspath(self, files: List[str]) -> None:
if not files:
return
results = []
for filename in files:
results.append(filename)
logger.info("\n".join(sorted(results)))
if files:
logger.info("\n".join(sorted(files)))
def remove_cache_items(self, options: Values, args: List[Any]) -> None:
if len(args) > 1:
@@ -195,8 +195,11 @@ class CacheCommand(Command):
return os.path.join(options.cache_dir, subdir)
def _find_http_files(self, options: Values) -> List[str]:
http_dir = self._cache_dir(options, "http")
return filesystem.find_files(http_dir, "*")
old_http_dir = self._cache_dir(options, "http")
new_http_dir = self._cache_dir(options, "http-v2")
return filesystem.find_files(old_http_dir, "*") + filesystem.find_files(
new_http_dir, "*"
)
def _find_wheels(self, options: Values, pattern: str) -> List[str]:
wheel_dir = self._cache_dir(options, "wheels")
@@ -23,9 +23,18 @@ COMPLETION_SCRIPTS = {
""",
"zsh": """
#compdef -P pip[0-9.]#
compadd $( COMP_WORDS="$words[*]" \\
COMP_CWORD=$((CURRENT-1)) \\
PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )
__pip() {{
compadd $( COMP_WORDS="$words[*]" \\
COMP_CWORD=$((CURRENT-1)) \\
PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )
}}
if [[ $zsh_eval_context[-1] == loadautofunc ]]; then
# autoload from fpath, call function directly
__pip "$@"
else
# eval/source/. command, register function for later
compdef __pip -P 'pip[0-9.]#'
fi
""",
"fish": """
function __fish_complete_pip
+13 -9
View File
@@ -46,22 +46,29 @@ def create_vendor_txt_map() -> Dict[str, str]:
return dict(line.split("==", 1) for line in lines)
def get_module_from_module_name(module_name: str) -> ModuleType:
def get_module_from_module_name(module_name: str) -> Optional[ModuleType]:
# Module name can be uppercase in vendor.txt for some reason...
module_name = module_name.lower().replace("-", "_")
# PATCH: setuptools is actually only pkg_resources.
if module_name == "setuptools":
module_name = "pkg_resources"
__import__(f"pipenv.patched.pip._vendor.{module_name}", globals(), locals(), level=0)
return getattr(pipenv.patched.pip._vendor, module_name)
try:
__import__(f"pipenv.patched.pip._vendor.{module_name}", globals(), locals(), level=0)
return getattr(pipenv.patched.pip._vendor, module_name)
except ImportError:
# We allow 'truststore' to fail to import due
# to being unavailable on Python 3.9 and earlier.
if module_name == "truststore" and sys.version_info < (3, 10):
return None
raise
def get_vendor_version_from_module(module_name: str) -> Optional[str]:
module = get_module_from_module_name(module_name)
version = getattr(module, "__version__", None)
if not version:
if module and not version:
# Try to find version in debundled module info.
assert module.__file__ is not None
env = get_environment([os.path.dirname(module.__file__)])
@@ -105,7 +112,7 @@ def show_tags(options: Values) -> None:
tag_limit = 10
target_python = make_target_python(options)
tags = target_python.get_tags()
tags = target_python.get_sorted_tags()
# Display the target options that were explicitly provided.
formatted_target = target_python.format_given()
@@ -134,10 +141,7 @@ def show_tags(options: Values) -> None:
def ca_bundle_info(config: Configuration) -> str:
levels = set()
for key, _ in config.items():
levels.add(key.split(".")[0])
levels = {key.split(".", 1)[0] for key, _ in config.items()}
if not levels:
return "Not specified"
@@ -501,7 +501,7 @@ class InstallCommand(RequirementCommand):
show_traceback,
options.use_user_site,
)
logger.error(message, exc_info=show_traceback) # noqa
logger.error(message, exc_info=show_traceback)
return ERROR
@@ -595,7 +595,7 @@ class InstallCommand(RequirementCommand):
"source of the following dependency conflicts."
)
else:
assert resolver_variant == "2020-resolver"
assert resolver_variant == "resolvelib"
parts.append(
"pip's dependency resolver does not currently take into account "
"all the packages that are installed. This behaviour is the "
@@ -628,7 +628,7 @@ class InstallCommand(RequirementCommand):
requirement=req,
dep_name=dep_name,
dep_version=dep_version,
you=("you" if resolver_variant == "2020-resolver" else "you'll"),
you=("you" if resolver_variant == "resolvelib" else "you'll"),
)
parts.append(message)
@@ -297,7 +297,7 @@ class ListCommand(IndexGroupCommand):
# Create and add a separator.
if len(data) > 0:
pkg_strings.insert(1, " ".join(map(lambda x: "-" * x, sizes)))
pkg_strings.insert(1, " ".join("-" * x for x in sizes))
for val in pkg_strings:
write_output(val)
@@ -1,4 +1,5 @@
import abc
from typing import Optional
from pipenv.patched.pip._internal.index.package_finder import PackageFinder
from pipenv.patched.pip._internal.metadata.base import BaseDistribution
@@ -19,12 +20,23 @@ class AbstractDistribution(metaclass=abc.ABCMeta):
- we must be able to create a Distribution object exposing the
above metadata.
- if we need to do work in the build tracker, we must be able to generate a unique
string to identify the requirement in the build tracker.
"""
def __init__(self, req: InstallRequirement) -> None:
super().__init__()
self.req = req
@abc.abstractproperty
def build_tracker_id(self) -> Optional[str]:
"""A string that uniquely identifies this requirement to the build tracker.
If None, then this dist has no work to do in the build tracker, and
``.prepare_distribution_metadata()`` will not be called."""
raise NotImplementedError()
@abc.abstractmethod
def get_metadata_distribution(self) -> BaseDistribution:
raise NotImplementedError()
@@ -1,3 +1,5 @@
from typing import Optional
from pipenv.patched.pip._internal.distributions.base import AbstractDistribution
from pipenv.patched.pip._internal.index.package_finder import PackageFinder
from pipenv.patched.pip._internal.metadata import BaseDistribution
@@ -10,6 +12,10 @@ class InstalledDistribution(AbstractDistribution):
been computed.
"""
@property
def build_tracker_id(self) -> Optional[str]:
return None
def get_metadata_distribution(self) -> BaseDistribution:
assert self.req.satisfied_by is not None, "not actually installed"
return self.req.satisfied_by
@@ -1,5 +1,5 @@
import logging
from typing import Iterable, Set, Tuple
from typing import Iterable, Optional, Set, Tuple
from pipenv.patched.pip._internal.build_env import BuildEnvironment
from pipenv.patched.pip._internal.distributions.base import AbstractDistribution
@@ -18,6 +18,12 @@ class SourceDistribution(AbstractDistribution):
generated, either using PEP 517 or using the legacy `setup.py egg_info`.
"""
@property
def build_tracker_id(self) -> Optional[str]:
"""Identify this requirement uniquely by its link."""
assert self.req.link
return self.req.link.url_without_fragment
def get_metadata_distribution(self) -> BaseDistribution:
return self.req.get_dist()
@@ -1,3 +1,5 @@
from typing import Optional
from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name
from pipenv.patched.pip._internal.distributions.base import AbstractDistribution
@@ -15,6 +17,10 @@ class WheelDistribution(AbstractDistribution):
This does not need any preparation as wheels can be directly unpacked.
"""
@property
def build_tracker_id(self) -> Optional[str]:
return None
def get_metadata_distribution(self) -> BaseDistribution:
"""Loads the metadata from the wheel file into memory and returns a
Distribution that uses it, not relying on the wheel file or
@@ -202,7 +202,7 @@ class LinkEvaluator:
reason = f"wrong project name (not {self.project_name})"
return (LinkType.different_project, reason)
supported_tags = self._target_python.get_tags()
supported_tags = self._target_python.get_unsorted_tags()
if not wheel.supported(supported_tags) and not self._ignore_compatibility:
# Include the wheel's tags in the reason string to
# simplify troubleshooting compatibility issues.
@@ -418,7 +418,7 @@ class CandidateEvaluator:
if specifier is None:
specifier = specifiers.SpecifierSet()
supported_tags = target_python.get_tags()
supported_tags = target_python.get_sorted_tags()
return cls(
project_name=project_name,
@@ -494,7 +494,7 @@ class CandidateEvaluator:
def _sort_key(
self,
candidate: InstallationCandidate,
ignore_compatibility: bool = True
ignore_compatibility: bool = True,
) -> CandidateSortingKey:
"""
Function to pass as the `key` argument to a call to sorted() to sort
@@ -89,7 +89,7 @@ def distutils_scheme(
# finalize_options(); we only want to override here if the user
# has explicitly requested it hence going back to the config
if "install_lib" in d.get_option_dict("install"):
scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib))
scheme.update({"purelib": i.install_lib, "platlib": i.install_lib})
if running_under_virtualenv():
if home:
@@ -9,7 +9,7 @@ from pipenv.patched.pip._internal.utils.misc import strtobool
from .base import BaseDistribution, BaseEnvironment, FilesystemWheel, MemoryWheel, Wheel
if TYPE_CHECKING:
from typing import Protocol
from typing import Literal, Protocol
else:
Protocol = object
@@ -50,6 +50,7 @@ def _should_use_importlib_metadata() -> bool:
class Backend(Protocol):
NAME: 'Literal["importlib", "pkg_resources"]'
Distribution: Type[BaseDistribution]
Environment: Type[BaseEnvironment]
+23 -9
View File
@@ -24,7 +24,7 @@ from typing import (
from pipenv.patched.pip._vendor.packaging.requirements import Requirement
from pipenv.patched.pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
from pipenv.patched.pip._vendor.packaging.utils import NormalizedName
from pipenv.patched.pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pipenv.patched.pip._vendor.packaging.version import LegacyVersion, Version
from pipenv.patched.pip._internal.exceptions import NoneMetadataError
@@ -37,7 +37,6 @@ from pipenv.patched.pip._internal.models.direct_url import (
from pipenv.patched.pip._internal.utils.compat import stdlib_pkgs # TODO: Move definition here.
from pipenv.patched.pip._internal.utils.egg_link import egg_link_path_from_sys_path
from pipenv.patched.pip._internal.utils.misc import is_local, normalize_path
from pipenv.patched.pip._internal.utils.packaging import safe_extra
from pipenv.patched.pip._internal.utils.urls import url_to_path
from ._json import msg_to_json
@@ -460,6 +459,19 @@ class BaseDistribution(Protocol):
For modern .dist-info distributions, this is the collection of
"Provides-Extra:" entries in distribution metadata.
The return value of this function is not particularly useful other than
display purposes due to backward compatibility issues and the extra
names being poorly normalized prior to PEP 685. If you want to perform
logic operations on extras, use :func:`is_extra_provided` instead.
"""
raise NotImplementedError()
def is_extra_provided(self, extra: str) -> bool:
"""Check whether an extra is provided by this distribution.
This is needed mostly for compatibility issues with pkg_resources not
following the extra normalization rules defined in PEP 685.
"""
raise NotImplementedError()
@@ -537,10 +549,11 @@ class BaseDistribution(Protocol):
"""Get extras from the egg-info directory."""
known_extras = {""}
for entry in self._iter_requires_txt_entries():
if entry.extra in known_extras:
extra = canonicalize_name(entry.extra)
if extra in known_extras:
continue
known_extras.add(entry.extra)
yield entry.extra
known_extras.add(extra)
yield extra
def _iter_egg_info_dependencies(self) -> Iterable[str]:
"""Get distribution dependencies from the egg-info directory.
@@ -556,10 +569,11 @@ class BaseDistribution(Protocol):
all currently available PEP 517 backends, although not standardized.
"""
for entry in self._iter_requires_txt_entries():
if entry.extra and entry.marker:
marker = f'({entry.marker}) and extra == "{safe_extra(entry.extra)}"'
elif entry.extra:
marker = f'extra == "{safe_extra(entry.extra)}"'
extra = canonicalize_name(entry.extra)
if extra and entry.marker:
marker = f'({entry.marker}) and extra == "{extra}"'
elif extra:
marker = f'extra == "{extra}"'
elif entry.marker:
marker = entry.marker
else:
@@ -1,4 +1,6 @@
from ._dists import Distribution
from ._envs import Environment
__all__ = ["Distribution", "Environment"]
__all__ = ["NAME", "Distribution", "Environment"]
NAME = "importlib"
@@ -27,7 +27,6 @@ from pipenv.patched.pip._internal.metadata.base import (
Wheel,
)
from pipenv.patched.pip._internal.utils.misc import normalize_path
from pipenv.patched.pip._internal.utils.packaging import safe_extra
from pipenv.patched.pip._internal.utils.temp_dir import TempDirectory
from pipenv.patched.pip._internal.utils.wheel import parse_wheel, read_wheel_metadata_file
@@ -208,12 +207,16 @@ class Distribution(BaseDistribution):
return cast(email.message.Message, self._dist.metadata)
def iter_provided_extras(self) -> Iterable[str]:
return (
safe_extra(extra) for extra in self.metadata.get_all("Provides-Extra", [])
return self.metadata.get_all("Provides-Extra", [])
def is_extra_provided(self, extra: str) -> bool:
return any(
canonicalize_name(provided_extra) == canonicalize_name(extra)
for provided_extra in self.metadata.get_all("Provides-Extra", [])
)
def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
contexts: Sequence[Dict[str, str]] = [{"extra": safe_extra(e)} for e in extras]
contexts: Sequence[Dict[str, str]] = [{"extra": e} for e in extras]
for req_string in self.metadata.get_all("Requires-Dist", []):
req = Requirement(req_string)
if not req.marker:
@@ -151,7 +151,8 @@ def _emit_egg_deprecation(location: Optional[str]) -> None:
deprecated(
reason=f"Loading egg at {location} is deprecated.",
replacement="to use pip for package installation.",
gone_in="23.3",
gone_in="24.3",
issue=12330,
)
@@ -24,8 +24,12 @@ from .base import (
Wheel,
)
__all__ = ["NAME", "Distribution", "Environment"]
logger = logging.getLogger(__name__)
NAME = "pkg_resources"
class EntryPoint(NamedTuple):
name: str
@@ -212,12 +216,16 @@ class Distribution(BaseDistribution):
def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
if extras: # pkg_resources raises on invalid extras, so we sanitize.
extras = frozenset(extras).intersection(self._dist.extras)
extras = frozenset(pkg_resources.safe_extra(e) for e in extras)
extras = extras.intersection(self._dist.extras)
return self._dist.requires(extras)
def iter_provided_extras(self) -> Iterable[str]:
return self._dist.extras
def is_extra_provided(self, extra: str) -> bool:
return pkg_resources.safe_extra(extra) in self._dist.extras
class Environment(BaseEnvironment):
def __init__(self, ws: pkg_resources.WorkingSet) -> None:
@@ -23,6 +23,9 @@ class InstallationReport:
# includes editable requirements), and false if the requirement was
# downloaded from a PEP 503 index or --find-links.
"is_direct": ireq.is_direct,
# is_yanked is true if the requirement was yanked from the index, but
# was still selected by pip to conform to PEP 592.
"is_yanked": ireq.link.is_yanked if ireq.link else False,
# requested is true if the requirement was specified by the user (aka
# top level requirement), and false if it was installed as a dependency of a
# requirement. https://peps.python.org/pep-0376/#requested
@@ -33,7 +36,7 @@ class InstallationReport:
}
if ireq.user_supplied and ireq.extras:
# For top level requirements, the list of requested extras, if any.
res["requested_extras"] = list(sorted(ireq.extras))
res["requested_extras"] = sorted(ireq.extras)
return res
def to_dict(self) -> Dict[str, Any]:
@@ -1,5 +1,5 @@
import sys
from typing import List, Optional, Tuple
from typing import List, Optional, Set, Tuple
from pipenv.patched.pip._vendor.packaging.tags import Tag
@@ -22,6 +22,7 @@ class TargetPython:
"py_version",
"py_version_info",
"_valid_tags",
"_valid_tags_set",
]
def __init__(
@@ -61,8 +62,9 @@ class TargetPython:
self.py_version = py_version
self.py_version_info = py_version_info
# This is used to cache the return value of get_tags().
# This is used to cache the return value of get_(un)sorted_tags.
self._valid_tags: Optional[List[Tag]] = None
self._valid_tags_set: Optional[Set[Tag]] = None
def format_given(self) -> str:
"""
@@ -84,7 +86,7 @@ class TargetPython:
f"{key}={value!r}" for key, value in key_values if value is not None
)
def get_tags(self) -> List[Tag]:
def get_sorted_tags(self) -> List[Tag]:
"""
Return the supported PEP 425 tags to check wheel candidates against.
@@ -108,3 +110,13 @@ class TargetPython:
self._valid_tags = tags
return self._valid_tags
def get_unsorted_tags(self) -> Set[Tag]:
"""Exactly the same as get_sorted_tags, but returns a set.
This is important for performance.
"""
if self._valid_tags_set is None:
self._valid_tags_set = set(self.get_sorted_tags())
return self._valid_tags_set
+25 -8
View File
@@ -3,10 +3,11 @@
import os
from contextlib import contextmanager
from typing import Generator, Optional
from datetime import datetime
from typing import BinaryIO, Generator, Optional, Union
from pipenv.patched.pip._vendor.cachecontrol.cache import BaseCache
from pipenv.patched.pip._vendor.cachecontrol.caches import FileCache
from pipenv.patched.pip._vendor.cachecontrol.cache import SeparateBodyBaseCache
from pipenv.patched.pip._vendor.cachecontrol.caches import SeparateBodyFileCache
from pipenv.patched.pip._vendor.requests.models import Response
from pipenv.patched.pip._internal.utils.filesystem import adjacent_tmp_file, replace
@@ -28,7 +29,7 @@ def suppressed_cache_errors() -> Generator[None, None, None]:
pass
class SafeFileCache(BaseCache):
class SafeFileCache(SeparateBodyBaseCache):
"""
A file based cache which is safe to use even when the target directory may
not be accessible or writable.
@@ -43,7 +44,7 @@ class SafeFileCache(BaseCache):
# From cachecontrol.caches.file_cache.FileCache._fn, brought into our
# class for backwards-compatibility and to avoid using a non-public
# method.
hashed = FileCache.encode(name)
hashed = SeparateBodyFileCache.encode(name)
parts = list(hashed[:5]) + [hashed]
return os.path.join(self.directory, *parts)
@@ -53,17 +54,33 @@ class SafeFileCache(BaseCache):
with open(path, "rb") as f:
return f.read()
def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None:
path = self._get_cache_path(key)
def _write(self, path: str, data: bytes) -> None:
with suppressed_cache_errors():
ensure_dir(os.path.dirname(path))
with adjacent_tmp_file(path) as f:
f.write(value)
f.write(data)
replace(f.name, path)
def set(
self, key: str, value: bytes, expires: Union[int, datetime, None] = None
) -> None:
path = self._get_cache_path(key)
self._write(path, value)
def delete(self, key: str) -> None:
path = self._get_cache_path(key)
with suppressed_cache_errors():
os.remove(path)
with suppressed_cache_errors():
os.remove(path + ".body")
def get_body(self, key: str) -> Optional[BinaryIO]:
path = self._get_cache_path(key) + ".body"
with suppressed_cache_errors():
return open(path, "rb")
def set_body(self, key: str, body: bytes) -> None:
path = self._get_cache_path(key) + ".body"
self._write(path, body)
@@ -51,10 +51,22 @@ def get_build_tracker() -> Generator["BuildTracker", None, None]:
yield tracker
class TrackerId(str):
"""Uniquely identifying string provided to the build tracker."""
class BuildTracker:
"""Ensure that an sdist cannot request itself as a setup requirement.
When an sdist is prepared, it identifies its setup requirements in the
context of ``BuildTracker.track()``. If a requirement shows up recursively, this
raises an exception.
This stops fork bombs embedded in malicious packages."""
def __init__(self, root: str) -> None:
self._root = root
self._entries: Set[InstallRequirement] = set()
self._entries: Dict[TrackerId, InstallRequirement] = {}
logger.debug("Created build tracker: %s", self._root)
def __enter__(self) -> "BuildTracker":
@@ -69,16 +81,15 @@ class BuildTracker:
) -> None:
self.cleanup()
def _entry_path(self, link: Link) -> str:
hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest()
def _entry_path(self, key: TrackerId) -> str:
hashed = hashlib.sha224(key.encode()).hexdigest()
return os.path.join(self._root, hashed)
def add(self, req: InstallRequirement) -> None:
def add(self, req: InstallRequirement, key: TrackerId) -> None:
"""Add an InstallRequirement to build tracking."""
assert req.link
# Get the file to write information about this requirement.
entry_path = self._entry_path(req.link)
entry_path = self._entry_path(key)
# Try reading from the file. If it exists and can be read from, a build
# is already in progress, so a LookupError is raised.
@@ -92,33 +103,37 @@ class BuildTracker:
raise LookupError(message)
# If we're here, req should really not be building already.
assert req not in self._entries
assert key not in self._entries
# Start tracking this requirement.
with open(entry_path, "w", encoding="utf-8") as fp:
fp.write(str(req))
self._entries.add(req)
self._entries[key] = req
logger.debug("Added %s to build tracker %r", req, self._root)
def remove(self, req: InstallRequirement) -> None:
def remove(self, req: InstallRequirement, key: TrackerId) -> None:
"""Remove an InstallRequirement from build tracking."""
assert req.link
# Delete the created file and the corresponding entries.
os.unlink(self._entry_path(req.link))
self._entries.remove(req)
# Delete the created file and the corresponding entry.
os.unlink(self._entry_path(key))
del self._entries[key]
logger.debug("Removed %s from build tracker %r", req, self._root)
def cleanup(self) -> None:
for req in set(self._entries):
self.remove(req)
for key, req in list(self._entries.items()):
self.remove(req, key)
logger.debug("Removed build tracker: %r", self._root)
@contextlib.contextmanager
def track(self, req: InstallRequirement) -> Generator[None, None, None]:
self.add(req)
def track(self, req: InstallRequirement, key: str) -> Generator[None, None, None]:
"""Ensure that `key` cannot install itself as a setup requirement.
:raises LookupError: If `key` was already provided in a parent invocation of
the context introduced by this method."""
tracker_id = TrackerId(key)
self.add(req, tracker_id)
yield
self.remove(req)
self.remove(req, tracker_id)
@@ -168,7 +168,7 @@ def warn_legacy_versions_and_specifiers(package_set: PackageSet) -> None:
f"release a version with a conforming version number"
),
issue=12063,
gone_in="23.3",
gone_in="24.0",
)
for dep in package_details.dependencies:
if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier):
@@ -183,5 +183,5 @@ def warn_legacy_versions_and_specifiers(package_set: PackageSet) -> None:
f"release a version with a conforming dependency specifiers"
),
issue=12063,
gone_in="23.3",
gone_in="24.0",
)
@@ -267,9 +267,9 @@ def get_csv_rows_for_installed(
path = _fs_to_record_path(f, lib_dir)
digest, length = rehash(f)
installed_rows.append((path, digest, length))
for installed_record_path in installed.values():
installed_rows.append((installed_record_path, "", ""))
return installed_rows
return installed_rows + [
(installed_record_path, "", "") for installed_record_path in installed.values()
]
def get_console_script_specs(console: Dict[str, str]) -> List[str]:
@@ -4,10 +4,10 @@
# The following comment should be removed at some point in the future.
# mypy: strict-optional=False
import logging
import mimetypes
import os
import shutil
from pathlib import Path
from typing import Dict, Iterable, List, Optional
from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name
@@ -21,7 +21,6 @@ from pipenv.patched.pip._internal.exceptions import (
InstallationError,
MetadataInconsistent,
NetworkConnectionError,
PreviousBuildDirError,
VcsHashUnsupported,
)
from pipenv.patched.pip._internal.index.package_finder import PackageFinder
@@ -37,6 +36,7 @@ from pipenv.patched.pip._internal.network.lazy_wheel import (
from pipenv.patched.pip._internal.network.session import PipSession
from pipenv.patched.pip._internal.operations.build.build_tracker import BuildTracker
from pipenv.patched.pip._internal.req.req_install import InstallRequirement
from pipenv.patched.pip._internal.utils._log import getLogger
from pipenv.patched.pip._internal.utils.direct_url_helpers import (
direct_url_for_editable,
direct_url_from_link,
@@ -47,13 +47,13 @@ from pipenv.patched.pip._internal.utils.misc import (
display_path,
hash_file,
hide_url,
is_installable_dir,
redact_auth_from_requirement,
)
from pipenv.patched.pip._internal.utils.temp_dir import TempDirectory
from pipenv.patched.pip._internal.utils.unpacking import unpack_file
from pipenv.patched.pip._internal.vcs import vcs
logger = logging.getLogger(__name__)
logger = getLogger(__name__)
def _get_prepared_distribution(
@@ -65,10 +65,12 @@ def _get_prepared_distribution(
) -> BaseDistribution:
"""Prepare a distribution for installation."""
abstract_dist = make_distribution_for_install_requirement(req)
with build_tracker.track(req):
abstract_dist.prepare_distribution_metadata(
finder, build_isolation, check_build_deps
)
tracker_id = abstract_dist.build_tracker_id
if tracker_id is not None:
with build_tracker.track(req, tracker_id):
abstract_dist.prepare_distribution_metadata(
finder, build_isolation, check_build_deps
)
return abstract_dist.get_metadata_distribution()
@@ -276,7 +278,7 @@ class RequirementPreparer:
information = str(display_path(req.link.file_path))
else:
message = "Collecting %s"
information = str(req.req or req)
information = redact_auth_from_requirement(req.req) if req.req else str(req)
# If we used req.req, inject requirement source if available (this
# would already be included if we used req directly)
@@ -317,21 +319,7 @@ class RequirementPreparer:
autodelete=True,
parallel_builds=parallel_builds,
)
# If a checkout exists, it's unwise to keep going. version
# inconsistencies are logged later, but do not fail the
# installation.
# FIXME: this won't upgrade when there's an existing
# package unpacked in `req.source_dir`
# TODO: this check is now probably dead code
if is_installable_dir(req.source_dir):
raise PreviousBuildDirError(
"pip can't proceed with requirements '{}' due to a"
"pre-existing build directory ({}). This is likely "
"due to a previous installation that failed . pip is "
"being responsible and not assuming it can delete this. "
"Please delete it and try again.".format(req, req.source_dir)
)
req.ensure_pristine_source_checkout()
def _get_linked_req_hashes(self, req: InstallRequirement) -> Hashes:
# By the time this is called, the requirement's link should have
@@ -394,7 +382,7 @@ class RequirementPreparer:
if metadata_link is None:
return None
assert req.req is not None
logger.info(
logger.verbose(
"Obtaining dependency information for %s from %s",
req.req,
metadata_link,
@@ -479,20 +467,19 @@ class RequirementPreparer:
for link, (filepath, _) in batch_download:
logger.debug("Downloading link %s to %s", link, filepath)
req = links_to_fully_download[link]
# Record the downloaded file path so wheel reqs can extract a Distribution
# in .get_dist().
req.local_file_path = filepath
# TODO: This needs fixing for sdists
# This is an emergency fix for #11847, which reports that
# distributions get downloaded twice when metadata is loaded
# from a PEP 658 standalone metadata file. Setting _downloaded
# fixes this for wheels, but breaks the sdist case (tests
# test_download_metadata). As PyPI is currently only serving
# metadata for wheels, this is not an immediate issue.
# Fixing the problem properly looks like it will require a
# complete refactoring of the `prepare_linked_requirements_more`
# logic, and I haven't a clue where to start on that, so for now
# I have fixed the issue *just* for wheels.
if req.is_wheel:
self._downloaded[req.link.url] = filepath
# Record that the file is downloaded so we don't do it again in
# _prepare_linked_requirement().
self._downloaded[req.link.url] = filepath
# If this is an sdist, we need to unpack it after downloading, but the
# .source_dir won't be set up until we are in _prepare_linked_requirement().
# Add the downloaded archive to the install requirement to unpack after
# preparing the source dir.
if not req.is_wheel:
req.needs_unpacked_archive(Path(filepath))
# This step is necessary to ensure all lazy wheels are processed
# successfully by the 'download', 'wheel', and 'install' commands.
@@ -8,10 +8,11 @@ These are meant to be used elsewhere within pip to create instances of
InstallRequirement.
"""
import copy
import logging
import os
import re
from typing import Dict, List, Optional, Set, Tuple, Union
from typing import Collection, Dict, List, Optional, Set, Tuple, Union
from pipenv.patched.pip._vendor.packaging.markers import Marker
from pipenv.patched.pip._vendor.packaging.requirements import InvalidRequirement, Requirement
@@ -57,6 +58,31 @@ def convert_extras(extras: Optional[str]) -> Set[str]:
return get_requirement("placeholder" + extras.lower()).extras
def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requirement:
"""
Returns a new requirement based on the given one, with the supplied extras. If the
given requirement already has extras those are replaced (or dropped if no new extras
are given).
"""
match: Optional[re.Match[str]] = re.fullmatch(
# see https://peps.python.org/pep-0508/#complete-grammar
r"([\w\t .-]+)(\[[^\]]*\])?(.*)",
str(req),
flags=re.ASCII,
)
# ireq.req is a valid requirement so the regex should always match
assert (
match is not None
), f"regex match on requirement {req} failed, this should never happen"
pre: Optional[str] = match.group(1)
post: Optional[str] = match.group(3)
assert (
pre is not None and post is not None
), f"regex group selection for requirement {req} failed, this should never happen"
extras: str = "[%s]" % ",".join(sorted(new_extras)) if new_extras else ""
return Requirement(f"{pre}{extras}{post}")
def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
"""Parses an editable requirement into:
- a requirement name
@@ -504,3 +530,47 @@ def install_req_from_link_and_ireq(
config_settings=ireq.config_settings,
user_supplied=ireq.user_supplied,
)
def install_req_drop_extras(ireq: InstallRequirement) -> InstallRequirement:
"""
Creates a new InstallationRequirement using the given template but without
any extras. Sets the original requirement as the new one's parent
(comes_from).
"""
return InstallRequirement(
req=(
_set_requirement_extras(ireq.req, set()) if ireq.req is not None else None
),
comes_from=ireq,
editable=ireq.editable,
link=ireq.link,
markers=ireq.markers,
use_pep517=ireq.use_pep517,
isolated=ireq.isolated,
global_options=ireq.global_options,
hash_options=ireq.hash_options,
constraint=ireq.constraint,
extras=[],
config_settings=ireq.config_settings,
user_supplied=ireq.user_supplied,
permit_editable_wheels=ireq.permit_editable_wheels,
)
def install_req_extend_extras(
ireq: InstallRequirement,
extras: Collection[str],
) -> InstallRequirement:
"""
Returns a copy of an installation requirement with some additional extras.
Makes a shallow copy of the ireq object.
"""
result = copy.copy(ireq)
result.extras = {*ireq.extras, *extras}
result.req = (
_set_requirement_extras(ireq.req, result.extras)
if ireq.req is not None
else None
)
return result
+59 -19
View File
@@ -1,6 +1,3 @@
# The following comment should be removed at some point in the future.
# mypy: strict-optional=False
import functools
import logging
import os
@@ -9,6 +6,7 @@ import sys
import uuid
import zipfile
from optparse import Values
from pathlib import Path
from typing import Any, Collection, Dict, Iterable, List, Optional, Sequence, Union
from pipenv.patched.pip._vendor.packaging.markers import Marker
@@ -20,7 +18,7 @@ from pipenv.patched.pip._vendor.packaging.version import parse as parse_version
from pipenv.patched.pip._vendor.pyproject_hooks import BuildBackendHookCaller
from pipenv.patched.pip._internal.build_env import BuildEnvironment, NoOpBuildEnvironment
from pipenv.patched.pip._internal.exceptions import InstallationError
from pipenv.patched.pip._internal.exceptions import InstallationError, PreviousBuildDirError
from pipenv.patched.pip._internal.locations import get_scheme
from pipenv.patched.pip._internal.metadata import (
BaseDistribution,
@@ -50,11 +48,14 @@ from pipenv.patched.pip._internal.utils.misc import (
backup_dir,
display_path,
hide_url,
is_installable_dir,
redact_auth_from_requirement,
redact_auth_from_url,
)
from pipenv.patched.pip._internal.utils.packaging import safe_extra
from pipenv.patched.pip._internal.utils.subprocess import runner_with_spinner_message
from pipenv.patched.pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
from pipenv.patched.pip._internal.utils.unpacking import unpack_file
from pipenv.patched.pip._internal.utils.virtualenv import running_under_virtualenv
from pipenv.patched.pip._internal.vcs import vcs
@@ -128,7 +129,7 @@ class InstallRequirement:
if extras:
self.extras = extras
elif req:
self.extras = {safe_extra(extra) for extra in req.extras}
self.extras = req.extras
else:
self.extras = set()
if markers is None and req:
@@ -183,9 +184,12 @@ class InstallRequirement:
# This requirement needs more preparation before it can be built
self.needs_more_preparation = False
# This requirement needs to be unpacked before it can be installed.
self._archive_source: Optional[Path] = None
def __str__(self) -> str:
if self.req:
s = str(self.req)
s = redact_auth_from_requirement(self.req)
if self.link:
s += " from {}".format(redact_auth_from_url(self.link.url))
elif self.link:
@@ -244,6 +248,7 @@ class InstallRequirement:
@property
def specifier(self) -> SpecifierSet:
assert self.req is not None
return self.req.specifier
@property
@@ -257,7 +262,8 @@ class InstallRequirement:
For example, some-package==1.2 is pinned; some-package>1.2 is not.
"""
specifiers = self.specifier
assert self.req is not None
specifiers = self.req.specifier
return len(specifiers) == 1 and next(iter(specifiers)).operator in {"==", "==="}
def match_markers(self, extras_requested: Optional[Iterable[str]] = None) -> bool:
@@ -267,7 +273,12 @@ class InstallRequirement:
extras_requested = ("",)
if self.markers is not None:
return any(
self.markers.evaluate({"extra": extra}) for extra in extras_requested
self.markers.evaluate({"extra": extra})
# TODO: Remove these two variants when packaging is upgraded to
# support the marker comparison logic specified in PEP 685.
or self.markers.evaluate({"extra": safe_extra(extra)})
or self.markers.evaluate({"extra": canonicalize_name(extra)})
for extra in extras_requested
)
else:
return True
@@ -305,6 +316,7 @@ class InstallRequirement:
else:
link = None
if link and link.hash:
assert link.hash_name is not None
good_hashes.setdefault(link.hash_name, []).append(link.hash)
return Hashes(good_hashes)
@@ -314,6 +326,7 @@ class InstallRequirement:
return None
s = str(self.req)
if self.comes_from:
comes_from: Optional[str]
if isinstance(self.comes_from, str):
comes_from = self.comes_from
else:
@@ -345,7 +358,7 @@ class InstallRequirement:
# When parallel builds are enabled, add a UUID to the build directory
# name so multiple builds do not interfere with each other.
dir_name: str = canonicalize_name(self.name)
dir_name: str = canonicalize_name(self.req.name)
if parallel_builds:
dir_name = f"{dir_name}_{uuid.uuid4().hex}"
@@ -388,6 +401,7 @@ class InstallRequirement:
)
def warn_on_mismatching_name(self) -> None:
assert self.req is not None
metadata_name = canonicalize_name(self.metadata["Name"])
if canonicalize_name(self.req.name) == metadata_name:
# Everything is fine.
@@ -457,6 +471,7 @@ class InstallRequirement:
# Things valid for sdists
@property
def unpacked_source_directory(self) -> str:
assert self.source_dir, f"No source dir for {self}"
return os.path.join(
self.source_dir, self.link and self.link.subdirectory_fragment or ""
)
@@ -500,7 +515,7 @@ class InstallRequirement:
"to use --use-pep517 or add a "
"pyproject.toml file to the project"
),
gone_in="23.3",
gone_in="24.0",
)
self.use_pep517 = False
return
@@ -544,7 +559,7 @@ class InstallRequirement:
Under PEP 517 and PEP 660, call the backend hook to prepare the metadata.
Under legacy processing, call setup.py egg-info.
"""
assert self.source_dir
assert self.source_dir, f"No source dir for {self}"
details = self.name or f"from {self.link}"
if self.use_pep517:
@@ -593,8 +608,10 @@ class InstallRequirement:
if self.metadata_directory:
return get_directory_distribution(self.metadata_directory)
elif self.local_file_path and self.is_wheel:
assert self.req is not None
return get_wheel_distribution(
FilesystemWheel(self.local_file_path), canonicalize_name(self.name)
FilesystemWheel(self.local_file_path),
canonicalize_name(self.req.name),
)
raise AssertionError(
f"InstallRequirement {self} has no metadata directory and no wheel: "
@@ -602,9 +619,9 @@ class InstallRequirement:
)
def assert_source_matches_version(self) -> None:
assert self.source_dir
assert self.source_dir, f"No source dir for {self}"
version = self.metadata["version"]
if self.req.specifier and version not in self.req.specifier:
if self.req and self.req.specifier and version not in self.req.specifier:
logger.warning(
"Requested %s, but installing version %s",
self,
@@ -641,6 +658,27 @@ class InstallRequirement:
parallel_builds=parallel_builds,
)
def needs_unpacked_archive(self, archive_source: Path) -> None:
assert self._archive_source is None
self._archive_source = archive_source
def ensure_pristine_source_checkout(self) -> None:
"""Ensure the source directory has not yet been built in."""
assert self.source_dir is not None
if self._archive_source is not None:
unpack_file(str(self._archive_source), self.source_dir)
elif is_installable_dir(self.source_dir):
# If a checkout exists, it's unwise to keep going.
# version inconsistencies are logged later, but do not fail
# the installation.
raise PreviousBuildDirError(
f"pip can't proceed with requirements '{self}' due to a "
f"pre-existing build directory ({self.source_dir}). This is likely "
"due to a previous installation that failed . pip is "
"being responsible and not assuming it can delete this. "
"Please delete it and try again."
)
# For editable installations
def update_editable(self) -> None:
if not self.link:
@@ -697,9 +735,10 @@ class InstallRequirement:
name = name.replace(os.path.sep, "/")
return name
assert self.req is not None
path = os.path.join(parentdir, path)
name = _clean_zip_name(path, rootdir)
return self.name + "/" + name
return self.req.name + "/" + name
def archive(self, build_dir: Optional[str]) -> None:
"""Saves archive to provided build_dir.
@@ -778,8 +817,9 @@ class InstallRequirement:
use_user_site: bool = False,
pycompile: bool = True,
) -> None:
assert self.req is not None
scheme = get_scheme(
self.name,
self.req.name,
user=use_user_site,
home=home,
root=root,
@@ -793,7 +833,7 @@ class InstallRequirement:
prefix=prefix,
home=home,
use_user_site=use_user_site,
name=self.name,
name=self.req.name,
setup_py_path=self.setup_py_path,
isolated=self.isolated,
build_env=self.build_env,
@@ -806,7 +846,7 @@ class InstallRequirement:
assert self.local_file_path
install_wheel(
self.name,
self.req.name,
self.local_file_path,
scheme=scheme,
req_description=str(self.req),
@@ -866,7 +906,7 @@ def check_legacy_setup_py_options(
reason="--build-option and --global-option are deprecated.",
issue=11859,
replacement="to use --config-settings",
gone_in="23.3",
gone_in="24.0",
)
logger.warning(
"Implying --no-binary=:all: due to the presence of "
+2 -2
View File
@@ -99,7 +99,7 @@ class RequirementSet:
"or contact the package author to fix the version number"
),
issue=12063,
gone_in="23.3",
gone_in="24.0",
)
for dep in req.get_dist().iter_dependencies():
if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier):
@@ -115,5 +115,5 @@ class RequirementSet:
"or contact the package author to fix the version number"
),
issue=12063,
gone_in="23.3",
gone_in="24.0",
)
@@ -274,7 +274,7 @@ class StashedUninstallPathSet:
def commit(self) -> None:
"""Commits the uninstall by removing stashed files."""
for _, save_dir in self._save_dirs.items():
for save_dir in self._save_dirs.values():
save_dir.cleanup()
self._moves = []
self._save_dirs = {}
@@ -1,7 +1,7 @@
from typing import FrozenSet, Iterable, Optional, Tuple, Union
from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet
from pipenv.patched.pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pipenv.patched.pip._vendor.packaging.utils import NormalizedName
from pipenv.patched.pip._vendor.packaging.version import LegacyVersion, Version
from pipenv.patched.pip._internal.models.link import Link, links_equivalent
@@ -12,11 +12,11 @@ CandidateLookup = Tuple[Optional["Candidate"], Optional[InstallRequirement]]
CandidateVersion = Union[LegacyVersion, Version]
def format_name(project: str, extras: FrozenSet[str]) -> str:
def format_name(project: NormalizedName, extras: FrozenSet[NormalizedName]) -> str:
if not extras:
return project
canonical_extras = sorted(canonicalize_name(e) for e in extras)
return "{}[{}]".format(project, ",".join(canonical_extras))
extras_expr = ",".join(sorted(extras))
return f"{project}[{extras_expr}]"
class Constraint:
@@ -241,7 +241,7 @@ class _InstallRequirementBackedCandidate(Candidate):
def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]:
requires = self.dist.iter_dependencies() if with_requires else ()
for r in requires:
yield self._factory.make_requirement_from_spec(str(r), self._ireq)
yield from self._factory.make_requirements_from_spec(str(r), self._ireq)
yield self._factory.make_requires_python_requirement(self.dist.requires_python)
def get_install_requirement(self) -> Optional[InstallRequirement]:
@@ -396,7 +396,7 @@ class AlreadyInstalledCandidate(Candidate):
if not with_requires:
return
for r in self.dist.iter_dependencies():
yield self._factory.make_requirement_from_spec(str(r), self._ireq)
yield from self._factory.make_requirements_from_spec(str(r), self._ireq)
def get_install_requirement(self) -> Optional[InstallRequirement]:
return None
@@ -431,9 +431,28 @@ class ExtrasCandidate(Candidate):
self,
base: BaseCandidate,
extras: FrozenSet[str],
*,
comes_from: Optional[InstallRequirement] = None,
) -> None:
"""
:param comes_from: the InstallRequirement that led to this candidate if it
differs from the base's InstallRequirement. This will often be the
case in the sense that this candidate's requirement has the extras
while the base's does not. Unlike the InstallRequirement backed
candidates, this requirement is used solely for reporting purposes,
it does not do any leg work.
"""
self.base = base
self.extras = extras
self.extras = frozenset(canonicalize_name(e) for e in extras)
# If any extras are requested in their non-normalized forms, keep track
# of their raw values. This is needed when we look up dependencies
# since PEP 685 has not been implemented for marker-matching, and using
# the non-normalized extra for lookup ensures the user can select a
# non-normalized extra in a package with its non-normalized form.
# TODO: Remove this attribute when packaging is upgraded to support the
# marker comparison logic specified in PEP 685.
self._unnormalized_extras = extras.difference(self.extras)
self._comes_from = comes_from if comes_from is not None else self.base._ireq
def __str__(self) -> str:
name, rest = str(self.base).split(" ", 1)
@@ -484,6 +503,50 @@ class ExtrasCandidate(Candidate):
def source_link(self) -> Optional[Link]:
return self.base.source_link
def _warn_invalid_extras(
self,
requested: FrozenSet[str],
valid: FrozenSet[str],
) -> None:
"""Emit warnings for invalid extras being requested.
This emits a warning for each requested extra that is not in the
candidate's ``Provides-Extra`` list.
"""
invalid_extras_to_warn = frozenset(
extra
for extra in requested
if extra not in valid
# If an extra is requested in an unnormalized form, skip warning
# about the normalized form being missing.
and extra in self.extras
)
if not invalid_extras_to_warn:
return
for extra in sorted(invalid_extras_to_warn):
logger.warning(
"%s %s does not provide the extra '%s'",
self.base.name,
self.version,
extra,
)
def _calculate_valid_requested_extras(self) -> FrozenSet[str]:
"""Get a list of valid extras requested by this candidate.
The user (or upstream dependant) may have specified extras that the
candidate doesn't support. Any unsupported extras are dropped, and each
cause a warning to be logged here.
"""
requested_extras = self.extras.union(self._unnormalized_extras)
valid_extras = frozenset(
extra
for extra in requested_extras
if self.base.dist.is_extra_provided(extra)
)
self._warn_invalid_extras(requested_extras, valid_extras)
return valid_extras
def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]:
factory = self.base._factory
@@ -493,24 +556,13 @@ class ExtrasCandidate(Candidate):
if not with_requires:
return
# The user may have specified extras that the candidate doesn't
# support. We ignore any unsupported extras here.
valid_extras = self.extras.intersection(self.base.dist.iter_provided_extras())
invalid_extras = self.extras.difference(self.base.dist.iter_provided_extras())
for extra in sorted(invalid_extras):
logger.warning(
"%s %s does not provide the extra '%s'",
self.base.name,
self.version,
extra,
)
valid_extras = self._calculate_valid_requested_extras()
for r in self.base.dist.iter_dependencies(valid_extras):
requirement = factory.make_requirement_from_spec(
str(r), self.base._ireq, valid_extras
yield from factory.make_requirements_from_spec(
str(r),
self._comes_from,
valid_extras,
)
if requirement:
yield requirement
def get_install_requirement(self) -> Optional[InstallRequirement]:
# We don't return anything here, because we always
@@ -62,6 +62,7 @@ from .requirements import (
ExplicitRequirement,
RequiresPythonRequirement,
SpecifierRequirement,
SpecifierWithoutExtrasRequirement,
UnsatisfiableRequirement,
)
@@ -112,7 +113,7 @@ class Factory:
self._editable_candidate_cache: Cache[EditableCandidate] = {}
self._installed_candidate_cache: Dict[str, AlreadyInstalledCandidate] = {}
self._extras_candidate_cache: Dict[
Tuple[int, FrozenSet[str]], ExtrasCandidate
Tuple[int, FrozenSet[NormalizedName]], ExtrasCandidate
] = {}
if not ignore_installed:
@@ -132,19 +133,23 @@ class Factory:
if not link.is_wheel:
return
wheel = Wheel(link.filename)
if wheel.supported(self._finder.target_python.get_tags()):
if wheel.supported(self._finder.target_python.get_unsorted_tags()):
return
msg = f"{link.filename} is not a supported wheel on this platform."
raise UnsupportedWheel(msg)
def _make_extras_candidate(
self, base: BaseCandidate, extras: FrozenSet[str]
self,
base: BaseCandidate,
extras: FrozenSet[str],
*,
comes_from: Optional[InstallRequirement] = None,
) -> ExtrasCandidate:
cache_key = (id(base), extras)
cache_key = (id(base), frozenset(canonicalize_name(e) for e in extras))
try:
candidate = self._extras_candidate_cache[cache_key]
except KeyError:
candidate = ExtrasCandidate(base, extras)
candidate = ExtrasCandidate(base, extras, comes_from=comes_from)
self._extras_candidate_cache[cache_key] = candidate
return candidate
@@ -161,7 +166,7 @@ class Factory:
self._installed_candidate_cache[dist.canonical_name] = base
if not extras:
return base
return self._make_extras_candidate(base, extras)
return self._make_extras_candidate(base, extras, comes_from=template)
def _make_candidate_from_link(
self,
@@ -223,7 +228,7 @@ class Factory:
if not extras:
return base
return self._make_extras_candidate(base, extras)
return self._make_extras_candidate(base, extras, comes_from=template)
def _iter_found_candidates(
self,
@@ -385,16 +390,21 @@ class Factory:
if ireq is not None:
ireqs.append(ireq)
# If the current identifier contains extras, add explicit candidates
# from entries from extra-less identifier.
# If the current identifier contains extras, add requires and explicit
# candidates from entries from extra-less identifier.
with contextlib.suppress(InvalidRequirement):
parsed_requirement = get_requirement(identifier)
explicit_candidates.update(
self._iter_explicit_candidates_from_base(
requirements.get(parsed_requirement.name, ()),
frozenset(parsed_requirement.extras),
),
)
if parsed_requirement.name != identifier:
explicit_candidates.update(
self._iter_explicit_candidates_from_base(
requirements.get(parsed_requirement.name, ()),
frozenset(parsed_requirement.extras),
),
)
for req in requirements.get(parsed_requirement.name, []):
_, ireq = req.get_candidate_lookup()
if ireq is not None:
ireqs.append(ireq)
# Add explicit candidates from constraints. We only do this if there are
# known ireqs, which represent requirements not already explicit. If
@@ -437,37 +447,49 @@ class Factory:
and all(req.is_satisfied_by(c) for req in requirements[identifier])
)
def _make_requirement_from_install_req(
def _make_requirements_from_install_req(
self, ireq: InstallRequirement, requested_extras: Iterable[str]
) -> Optional[Requirement]:
) -> Iterator[Requirement]:
"""
Returns requirement objects associated with the given InstallRequirement. In
most cases this will be a single object but the following special cases exist:
- the InstallRequirement has markers that do not apply -> result is empty
- the InstallRequirement has both a constraint and extras -> result is split
in two requirement objects: one with the constraint and one with the
extra. This allows centralized constraint handling for the base,
resulting in fewer candidate rejections.
"""
if not ireq.match_markers(requested_extras):
logger.info(
"Ignoring %s: markers '%s' don't match your environment",
ireq.name,
ireq.markers,
)
return None
if not ireq.link:
return SpecifierRequirement(ireq)
self._fail_if_link_is_unsupported_wheel(ireq.link)
cand = self._make_candidate_from_link(
ireq.link,
extras=frozenset(ireq.extras),
template=ireq,
name=canonicalize_name(ireq.name) if ireq.name else None,
version=None,
)
if cand is None:
# There's no way we can satisfy a URL requirement if the underlying
# candidate fails to build. An unnamed URL must be user-supplied, so
# we fail eagerly. If the URL is named, an unsatisfiable requirement
# can make the resolver do the right thing, either backtrack (and
# maybe find some other requirement that's buildable) or raise a
# ResolutionImpossible eventually.
if not ireq.name:
raise self._build_failures[ireq.link]
return UnsatisfiableRequirement(canonicalize_name(ireq.name))
return self.make_requirement_from_candidate(cand)
elif not ireq.link:
if ireq.extras and ireq.req is not None and ireq.req.specifier:
yield SpecifierWithoutExtrasRequirement(ireq)
yield SpecifierRequirement(ireq)
else:
self._fail_if_link_is_unsupported_wheel(ireq.link)
cand = self._make_candidate_from_link(
ireq.link,
extras=frozenset(ireq.extras),
template=ireq,
name=canonicalize_name(ireq.name) if ireq.name else None,
version=None,
)
if cand is None:
# There's no way we can satisfy a URL requirement if the underlying
# candidate fails to build. An unnamed URL must be user-supplied, so
# we fail eagerly. If the URL is named, an unsatisfiable requirement
# can make the resolver do the right thing, either backtrack (and
# maybe find some other requirement that's buildable) or raise a
# ResolutionImpossible eventually.
if not ireq.name:
raise self._build_failures[ireq.link]
yield UnsatisfiableRequirement(canonicalize_name(ireq.name))
else:
yield self.make_requirement_from_candidate(cand)
def collect_root_requirements(
self, root_ireqs: List[InstallRequirement]
@@ -488,15 +510,27 @@ class Factory:
else:
collected.constraints[name] = Constraint.from_ireq(ireq)
else:
req = self._make_requirement_from_install_req(
ireq,
requested_extras=(),
reqs = list(
self._make_requirements_from_install_req(
ireq,
requested_extras=(),
)
)
if req is None:
if not reqs:
continue
if ireq.user_supplied and req.name not in collected.user_requested:
collected.user_requested[req.name] = i
collected.requirements.append(req)
template = reqs[0]
if ireq.user_supplied and template.name not in collected.user_requested:
collected.user_requested[template.name] = i
collected.requirements.extend(reqs)
# Put requirements with extras at the end of the root requires. This does not
# affect resolvelib's picking preference but it does affect its initial criteria
# population: by putting extras at the end we enable the candidate finder to
# present resolvelib with a smaller set of candidates to resolvelib, already
# taking into account any non-transient constraints on the associated base. This
# means resolvelib will have fewer candidates to visit and reject.
# Python's list sort is stable, meaning relative order is kept for objects with
# the same key.
collected.requirements.sort(key=lambda r: r.name != r.project_name)
return collected
def make_requirement_from_candidate(
@@ -504,14 +538,23 @@ class Factory:
) -> ExplicitRequirement:
return ExplicitRequirement(candidate)
def make_requirement_from_spec(
def make_requirements_from_spec(
self,
specifier: str,
comes_from: Optional[InstallRequirement],
requested_extras: Iterable[str] = (),
) -> Optional[Requirement]:
) -> Iterator[Requirement]:
"""
Returns requirement objects associated with the given specifier. In most cases
this will be a single object but the following special cases exist:
- the specifier has markers that do not apply -> result is empty
- the specifier has both a constraint and extras -> result is split
in two requirement objects: one with the constraint and one with the
extra. This allows centralized constraint handling for the base,
resulting in fewer candidate rejections.
"""
ireq = self._make_install_req_from_spec(specifier, comes_from)
return self._make_requirement_from_install_req(ireq, requested_extras)
return self._make_requirements_from_install_req(ireq, requested_extras)
def make_requires_python_requirement(
self,
@@ -603,8 +646,26 @@ class Factory:
cands = self._finder.find_all_candidates(req.project_name)
skipped_by_requires_python = self._finder.requires_python_skipped_reasons()
versions = [str(v) for v in sorted({c.version for c in cands})]
versions_set: Set[CandidateVersion] = set()
yanked_versions_set: Set[CandidateVersion] = set()
for c in cands:
is_yanked = c.link.is_yanked if c.link else False
if is_yanked:
yanked_versions_set.add(c.version)
else:
versions_set.add(c.version)
versions = [str(v) for v in sorted(versions_set)]
yanked_versions = [str(v) for v in sorted(yanked_versions_set)]
if yanked_versions:
# Saying "version X is yanked" isn't entirely accurate.
# https://github.com/pypa/pip/issues/11745#issuecomment-1402805842
logger.critical(
"Ignored the following yanked versions: %s",
", ".join(yanked_versions) or "none",
)
if skipped_by_requires_python:
logger.critical(
"Ignored the following versions that require a different python "
@@ -1,6 +1,7 @@
from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet
from pipenv.patched.pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pipenv.patched.pip._internal.req.constructors import install_req_drop_extras
from pipenv.patched.pip._internal.req.req_install import InstallRequirement
from .base import Candidate, CandidateLookup, Requirement, format_name
@@ -43,7 +44,7 @@ class SpecifierRequirement(Requirement):
def __init__(self, ireq: InstallRequirement) -> None:
assert ireq.link is None, "This is a link, not a specifier"
self._ireq = ireq
self._extras = frozenset(ireq.extras)
self._extras = frozenset(canonicalize_name(e) for e in self._ireq.extras)
def __str__(self) -> str:
return str(self._ireq.req)
@@ -92,6 +93,18 @@ class SpecifierRequirement(Requirement):
return spec.contains(candidate.version, prereleases=True)
class SpecifierWithoutExtrasRequirement(SpecifierRequirement):
"""
Requirement backed by an install requirement on a base package.
Trims extras from its install requirement if there are any.
"""
def __init__(self, ireq: InstallRequirement) -> None:
assert ireq.link is None, "This is a link, not a specifier"
self._ireq = install_req_drop_extras(ireq)
self._extras = frozenset(canonicalize_name(e) for e in self._ireq.extras)
class RequiresPythonRequirement(Requirement):
"""A requirement representing Requires-Python metadata."""
@@ -1,3 +1,4 @@
import contextlib
import functools
import logging
import os
@@ -11,6 +12,7 @@ from pipenv.patched.pip._vendor.resolvelib.structs import DirectedGraph
from pipenv.patched.pip._internal.cache import WheelCache
from pipenv.patched.pip._internal.index.package_finder import PackageFinder
from pipenv.patched.pip._internal.operations.prepare import RequirementPreparer
from pipenv.patched.pip._internal.req.constructors import install_req_extend_extras
from pipenv.patched.pip._internal.req.req_install import InstallRequirement
from pipenv.patched.pip._internal.req.req_set import RequirementSet
from pipenv.patched.pip._internal.resolution.base import BaseResolver, InstallRequirementProvider
@@ -19,6 +21,7 @@ from pipenv.patched.pip._internal.resolution.resolvelib.reporter import (
PipDebuggingReporter,
PipReporter,
)
from pipenv.patched.pip._internal.utils.packaging import get_requirement
from .base import Candidate, Requirement
from .factory import Factory
@@ -101,9 +104,24 @@ class Resolver(BaseResolver):
raise error from e
req_set = RequirementSet(check_supported_wheels=check_supported_wheels)
for candidate in result.mapping.values():
# process candidates with extras last to ensure their base equivalent is
# already in the req_set if appropriate.
# Python's sort is stable so using a binary key function keeps relative order
# within both subsets.
for candidate in sorted(
result.mapping.values(), key=lambda c: c.name != c.project_name
):
ireq = candidate.get_install_requirement()
if ireq is None:
if candidate.name != candidate.project_name:
# extend existing req's extras
with contextlib.suppress(KeyError):
req = req_set.get_requirement(candidate.project_name)
req_set.add_named_requirement(
install_req_extend_extras(
req, get_requirement(candidate.name).extras
)
)
continue
# Check if there is already an installation under the same name,
@@ -28,8 +28,7 @@ from pipenv.patched.pip._internal.utils.entrypoints import (
from pipenv.patched.pip._internal.utils.filesystem import adjacent_tmp_file, check_path_owner, replace
from pipenv.patched.pip._internal.utils.misc import ensure_dir
_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ"
_WEEK = datetime.timedelta(days=7)
logger = logging.getLogger(__name__)
@@ -73,12 +72,10 @@ class SelfCheckState:
if "pypi_version" not in self._state:
return None
seven_days_in_seconds = 7 * 24 * 60 * 60
# Determine if we need to refresh the state
last_check = datetime.datetime.strptime(self._state["last_check"], _DATE_FMT)
seconds_since_last_check = (current_time - last_check).total_seconds()
if seconds_since_last_check > seven_days_in_seconds:
last_check = datetime.datetime.fromisoformat(self._state["last_check"])
time_since_last_check = current_time - last_check
if time_since_last_check > _WEEK:
return None
return self._state["pypi_version"]
@@ -100,7 +97,7 @@ class SelfCheckState:
# Include the key so it's easy to tell which pip wrote the
# file.
"key": self.key,
"last_check": current_time.strftime(_DATE_FMT),
"last_check": current_time.isoformat(),
"pypi_version": pypi_version,
}
@@ -229,14 +226,14 @@ def pip_self_version_check(session: PipSession, options: optparse.Values) -> Non
try:
upgrade_prompt = _self_version_check_logic(
state=SelfCheckState(cache_dir=options.cache_dir),
current_time=datetime.datetime.utcnow(),
current_time=datetime.datetime.now(datetime.timezone.utc),
local_version=installed_dist.version,
get_remote_version=functools.partial(
_get_current_remote_pip_version, session, options
),
)
if upgrade_prompt is not None:
logger.warning("[present-rich] %s", upgrade_prompt)
logger.warning("%s", upgrade_prompt, extra={"rich": True})
except Exception:
logger.warning("There was an error checking the latest version of pip.")
logger.debug("See below for error", exc_info=True)
@@ -1,35 +0,0 @@
"""A helper module that injects SecureTransport, on import.
The import should be done as early as possible, to ensure all requests and
sessions (or whatever) are created after injecting SecureTransport.
Note that we only do the injection on macOS, when the linked OpenSSL is too
old to handle TLSv1.2.
"""
import sys
def inject_securetransport() -> None:
# Only relevant on macOS
if sys.platform != "darwin":
return
try:
import ssl
except ImportError:
return
# Checks for OpenSSL 1.0.1
if ssl.OPENSSL_VERSION_NUMBER >= 0x1000100F:
return
try:
from pipenv.patched.pip._vendor.urllib3.contrib import securetransport
except (ImportError, OSError):
return
securetransport.inject_into_urllib3()
inject_securetransport()
@@ -155,8 +155,8 @@ class RichPipStreamHandler(RichHandler):
# If we are given a diagnostic error to present, present it with indentation.
assert isinstance(record.args, tuple)
if record.msg == "[present-rich] %s" and len(record.args) == 1:
rich_renderable = record.args[0]
if getattr(record, "rich", False):
(rich_renderable,) = record.args
assert isinstance(
rich_renderable, (ConsoleRenderable, RichCast, str)
), f"{rich_renderable} is not rich-console-renderable"
+70 -16
View File
@@ -11,9 +11,11 @@ import stat
import sys
import sysconfig
import urllib.parse
from functools import partial
from io import StringIO
from itertools import filterfalse, tee, zip_longest
from types import TracebackType
from pathlib import Path
from types import FunctionType, TracebackType
from typing import (
Any,
BinaryIO,
@@ -33,6 +35,7 @@ from typing import (
cast,
)
from pipenv.patched.pip._vendor.packaging.requirements import Requirement
from pipenv.patched.pip._vendor.pyproject_hooks import BuildBackendHookCaller
from pipenv.patched.pip._vendor.tenacity import retry, stop_after_delay, wait_fixed
@@ -66,6 +69,8 @@ T = TypeVar("T")
ExcInfo = Tuple[Type[BaseException], BaseException, TracebackType]
VersionInfo = Tuple[int, int, int]
NetlocTuple = Tuple[str, Tuple[Optional[str], Optional[str]]]
OnExc = Callable[[FunctionType, Path, BaseException], Any]
OnErr = Callable[[FunctionType, Path, ExcInfo], Any]
def get_pip_version() -> str:
@@ -123,33 +128,75 @@ def get_prog() -> str:
# Retry every half second for up to 3 seconds
# Tenacity raises RetryError by default, explicitly raise the original exception
@retry(reraise=True, stop=stop_after_delay(3), wait=wait_fixed(0.5))
def rmtree(dir: str, ignore_errors: bool = False) -> None:
def rmtree(
dir: str,
ignore_errors: bool = False,
onexc: Optional[OnExc] = None,
) -> None:
if ignore_errors:
onexc = _onerror_ignore
if onexc is None:
onexc = _onerror_reraise
handler: OnErr = partial(
# `[func, path, Union[ExcInfo, BaseException]] -> Any` is equivalent to
# `Union[([func, path, ExcInfo] -> Any), ([func, path, BaseException] -> Any)]`.
cast(Union[OnExc, OnErr], rmtree_errorhandler),
onexc=onexc,
)
if sys.version_info >= (3, 12):
shutil.rmtree(dir, ignore_errors=ignore_errors, onexc=rmtree_errorhandler)
# See https://docs.python.org/3.12/whatsnew/3.12.html#shutil.
shutil.rmtree(dir, onexc=handler)
else:
shutil.rmtree(dir, ignore_errors=ignore_errors, onerror=rmtree_errorhandler)
shutil.rmtree(dir, onerror=handler)
def _onerror_ignore(*_args: Any) -> None:
pass
def _onerror_reraise(*_args: Any) -> None:
raise
def rmtree_errorhandler(
func: Callable[..., Any], path: str, exc_info: Union[ExcInfo, BaseException]
func: FunctionType,
path: Path,
exc_info: Union[ExcInfo, BaseException],
*,
onexc: OnExc = _onerror_reraise,
) -> None:
"""On Windows, the files in .svn are read-only, so when rmtree() tries to
remove them, an exception is thrown. We catch that here, remove the
read-only attribute, and hopefully continue without problems."""
"""
`rmtree` error handler to 'force' a file remove (i.e. like `rm -f`).
* If a file is readonly then it's write flag is set and operation is
retried.
* `onerror` is the original callback from `rmtree(... onerror=onerror)`
that is chained at the end if the "rm -f" still fails.
"""
try:
has_attr_readonly = not (os.stat(path).st_mode & stat.S_IWRITE)
st_mode = os.stat(path).st_mode
except OSError:
# it's equivalent to os.path.exists
return
if has_attr_readonly:
if not st_mode & stat.S_IWRITE:
# convert to read/write
os.chmod(path, stat.S_IWRITE)
# use the original function to repeat the operation
func(path)
return
else:
raise
try:
os.chmod(path, st_mode | stat.S_IWRITE)
except OSError:
pass
else:
# use the original function to repeat the operation
try:
func(path)
return
except OSError:
pass
if not isinstance(exc_info, BaseException):
_, exc_info, _ = exc_info
onexc(func, path, exc_info)
def display_path(path: str) -> str:
@@ -532,6 +579,13 @@ def redact_auth_from_url(url: str) -> str:
return _transform_url(url, _redact_netloc)[0]
def redact_auth_from_requirement(req: Requirement) -> str:
"""Replace the password in a given requirement url with ****."""
if not req.url:
return str(req)
return str(req).replace(req.url, redact_auth_from_url(req.url))
class HiddenText:
def __init__(self, secret: str, redacted: str) -> None:
self.secret = secret
@@ -209,7 +209,7 @@ def call_subprocess(
output_lines=all_output if not showing_subprocess else None,
)
if log_failed_cmd:
subprocess_logger.error("[present-rich] %s", error)
subprocess_logger.error("%s", error, extra={"rich": True})
subprocess_logger.verbose(
"[bold magenta]full command[/]: [blue]%s[/]",
escape(format_command_args(cmd)),
+52 -2
View File
@@ -3,8 +3,19 @@ import itertools
import logging
import os.path
import tempfile
import traceback
from contextlib import ExitStack, contextmanager
from typing import Any, Dict, Generator, Optional, TypeVar, Union
from pathlib import Path
from typing import (
Any,
Callable,
Dict,
Generator,
List,
Optional,
TypeVar,
Union,
)
from pipenv.patched.pip._internal.utils.misc import enum, rmtree
@@ -106,6 +117,7 @@ class TempDirectory:
delete: Union[bool, None, _Default] = _default,
kind: str = "temp",
globally_managed: bool = False,
ignore_cleanup_errors: bool = True,
):
super().__init__()
@@ -128,6 +140,7 @@ class TempDirectory:
self._deleted = False
self.delete = delete
self.kind = kind
self.ignore_cleanup_errors = ignore_cleanup_errors
if globally_managed:
assert _tempdir_manager is not None
@@ -170,7 +183,44 @@ class TempDirectory:
self._deleted = True
if not os.path.exists(self._path):
return
rmtree(self._path)
errors: List[BaseException] = []
def onerror(
func: Callable[..., Any],
path: Path,
exc_val: BaseException,
) -> None:
"""Log a warning for a `rmtree` error and continue"""
formatted_exc = "\n".join(
traceback.format_exception_only(type(exc_val), exc_val)
)
formatted_exc = formatted_exc.rstrip() # remove trailing new line
if func in (os.unlink, os.remove, os.rmdir):
logger.debug(
"Failed to remove a temporary file '%s' due to %s.\n",
path,
formatted_exc,
)
else:
logger.debug("%s failed with %s.", func.__qualname__, formatted_exc)
errors.append(exc_val)
if self.ignore_cleanup_errors:
try:
# first try with tenacity; retrying to handle ephemeral errors
rmtree(self._path, ignore_errors=False)
except OSError:
# last pass ignore/log all errors
rmtree(self._path, onexc=onerror)
if errors:
logger.warning(
"Failed to remove contents in a temporary directory '%s'.\n"
"You can safely remove it manually.",
self._path,
)
else:
rmtree(self._path)
class AdjacentTempDirectory(TempDirectory):
+1 -1
View File
@@ -101,7 +101,7 @@ class Git(VersionControl):
if not match:
logger.warning("Can't parse git version: %s", version)
return ()
return tuple(int(c) for c in match.groups())
return (int(match.group(1)), int(match.group(2)))
@classmethod
def get_current_branch(cls, location: str) -> Optional[str]:
@@ -31,7 +31,7 @@ class Mercurial(VersionControl):
@staticmethod
def get_base_rev_args(rev: str) -> List[str]:
return ["-r", rev]
return [f"-r={rev}"]
def fetch_new(
self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
+1
View File
@@ -117,4 +117,5 @@ if DEBUNDLED:
vendored("rich.traceback")
vendored("tenacity")
vendored("tomli")
vendored("truststore")
vendored("urllib3")
@@ -8,11 +8,21 @@ Make it easy to import from cachecontrol without long namespaces.
"""
__author__ = "Eric Larson"
__email__ = "eric@ionrock.org"
__version__ = "0.12.11"
__version__ = "0.13.1"
from .wrapper import CacheControl
from .adapter import CacheControlAdapter
from .controller import CacheController
from pipenv.patched.pip._vendor.cachecontrol.adapter import CacheControlAdapter
from pipenv.patched.pip._vendor.cachecontrol.controller import CacheController
from pipenv.patched.pip._vendor.cachecontrol.wrapper import CacheControl
__all__ = [
"__author__",
"__email__",
"__version__",
"CacheControlAdapter",
"CacheController",
"CacheControl",
]
import logging
logging.getLogger(__name__).addHandler(logging.NullHandler())
@@ -1,8 +1,11 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import logging
from argparse import ArgumentParser
from typing import TYPE_CHECKING
from pipenv.patched.pip._vendor import requests
@@ -10,16 +13,19 @@ from pipenv.patched.pip._vendor.cachecontrol.adapter import CacheControlAdapter
from pipenv.patched.pip._vendor.cachecontrol.cache import DictCache
from pipenv.patched.pip._vendor.cachecontrol.controller import logger
from argparse import ArgumentParser
if TYPE_CHECKING:
from argparse import Namespace
from pipenv.patched.pip._vendor.cachecontrol.controller import CacheController
def setup_logging():
def setup_logging() -> None:
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
logger.addHandler(handler)
def get_session():
def get_session() -> requests.Session:
adapter = CacheControlAdapter(
DictCache(), cache_etags=True, serializer=None, heuristic=None
)
@@ -27,17 +33,17 @@ def get_session():
sess.mount("http://", adapter)
sess.mount("https://", adapter)
sess.cache_controller = adapter.controller
sess.cache_controller = adapter.controller # type: ignore[attr-defined]
return sess
def get_args():
def get_args() -> Namespace:
parser = ArgumentParser()
parser.add_argument("url", help="The URL to try and cache")
return parser.parse_args()
def main(args=None):
def main() -> None:
args = get_args()
sess = get_session()
@@ -48,10 +54,13 @@ def main(args=None):
setup_logging()
# try setting the cache
sess.cache_controller.cache_response(resp.request, resp.raw)
cache_controller: CacheController = (
sess.cache_controller # type: ignore[attr-defined]
)
cache_controller.cache_response(resp.request, resp.raw)
# Now try to get it
if sess.cache_controller.cached_request(resp.request):
if cache_controller.cached_request(resp.request):
print("Cached!")
else:
print("Not cached :(")
@@ -1,16 +1,26 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import types
import functools
import types
import zlib
from typing import TYPE_CHECKING, Any, Collection, Mapping
from pipenv.patched.pip._vendor.requests.adapters import HTTPAdapter
from .controller import CacheController, PERMANENT_REDIRECT_STATUSES
from .cache import DictCache
from .filewrapper import CallbackFileWrapper
from pipenv.patched.pip._vendor.cachecontrol.cache import DictCache
from pipenv.patched.pip._vendor.cachecontrol.controller import PERMANENT_REDIRECT_STATUSES, CacheController
from pipenv.patched.pip._vendor.cachecontrol.filewrapper import CallbackFileWrapper
if TYPE_CHECKING:
from pipenv.patched.pip._vendor.requests import PreparedRequest, Response
from pipenv.patched.pip._vendor.urllib3 import HTTPResponse
from pipenv.patched.pip._vendor.cachecontrol.cache import BaseCache
from pipenv.patched.pip._vendor.cachecontrol.heuristics import BaseHeuristic
from pipenv.patched.pip._vendor.cachecontrol.serialize import Serializer
class CacheControlAdapter(HTTPAdapter):
@@ -18,16 +28,16 @@ class CacheControlAdapter(HTTPAdapter):
def __init__(
self,
cache=None,
cache_etags=True,
controller_class=None,
serializer=None,
heuristic=None,
cacheable_methods=None,
*args,
**kw
):
super(CacheControlAdapter, self).__init__(*args, **kw)
cache: BaseCache | None = None,
cache_etags: bool = True,
controller_class: type[CacheController] | None = None,
serializer: Serializer | None = None,
heuristic: BaseHeuristic | None = None,
cacheable_methods: Collection[str] | None = None,
*args: Any,
**kw: Any,
) -> None:
super().__init__(*args, **kw)
self.cache = DictCache() if cache is None else cache
self.heuristic = heuristic
self.cacheable_methods = cacheable_methods or ("GET",)
@@ -37,7 +47,16 @@ class CacheControlAdapter(HTTPAdapter):
self.cache, cache_etags=cache_etags, serializer=serializer
)
def send(self, request, cacheable_methods=None, **kw):
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: None | float | tuple[float, float] | tuple[float, None] = None,
verify: bool | str = True,
cert: (None | bytes | str | tuple[bytes | str, bytes | str]) = None,
proxies: Mapping[str, str] | None = None,
cacheable_methods: Collection[str] | None = None,
) -> Response:
"""
Send a request. Use the request information to see if it
exists in the cache and cache the response if we need to and can.
@@ -54,13 +73,17 @@ class CacheControlAdapter(HTTPAdapter):
# check for etags and add headers if appropriate
request.headers.update(self.controller.conditional_headers(request))
resp = super(CacheControlAdapter, self).send(request, **kw)
resp = super().send(request, stream, timeout, verify, cert, proxies)
return resp
def build_response(
self, request, response, from_cache=False, cacheable_methods=None
):
self,
request: PreparedRequest,
response: HTTPResponse,
from_cache: bool = False,
cacheable_methods: Collection[str] | None = None,
) -> Response:
"""
Build a response by making a request or using the cache.
@@ -102,36 +125,37 @@ class CacheControlAdapter(HTTPAdapter):
else:
# Wrap the response file with a wrapper that will cache the
# response when the stream has been consumed.
response._fp = CallbackFileWrapper(
response._fp,
response._fp = CallbackFileWrapper( # type: ignore[attr-defined]
response._fp, # type: ignore[attr-defined]
functools.partial(
self.controller.cache_response, request, response
),
)
if response.chunked:
super_update_chunk_length = response._update_chunk_length
super_update_chunk_length = response._update_chunk_length # type: ignore[attr-defined]
def _update_chunk_length(self):
def _update_chunk_length(self: HTTPResponse) -> None:
super_update_chunk_length()
if self.chunk_left == 0:
self._fp._close()
self._fp._close() # type: ignore[attr-defined]
response._update_chunk_length = types.MethodType(
response._update_chunk_length = types.MethodType( # type: ignore[attr-defined]
_update_chunk_length, response
)
resp = super(CacheControlAdapter, self).build_response(request, response)
resp: Response = super().build_response(request, response) # type: ignore[no-untyped-call]
# See if we should invalidate the cache.
if request.method in self.invalidating_methods and resp.ok:
assert request.url is not None
cache_url = self.controller.cache_url(request.url)
self.cache.delete(cache_url)
# Give the request a from_cache attr to let people use it
resp.from_cache = from_cache
resp.from_cache = from_cache # type: ignore[attr-defined]
return resp
def close(self):
def close(self) -> None:
self.cache.close()
super(CacheControlAdapter, self).close()
super().close() # type: ignore[no-untyped-call]
@@ -6,38 +6,46 @@
The cache object API for implementing caches. The default is a thread
safe in-memory dictionary.
"""
from __future__ import annotations
from threading import Lock
from typing import IO, TYPE_CHECKING, MutableMapping
if TYPE_CHECKING:
from datetime import datetime
class BaseCache(object):
def get(self, key):
class BaseCache:
def get(self, key: str) -> bytes | None:
raise NotImplementedError()
def set(self, key, value, expires=None):
def set(
self, key: str, value: bytes, expires: int | datetime | None = None
) -> None:
raise NotImplementedError()
def delete(self, key):
def delete(self, key: str) -> None:
raise NotImplementedError()
def close(self):
def close(self) -> None:
pass
class DictCache(BaseCache):
def __init__(self, init_dict=None):
def __init__(self, init_dict: MutableMapping[str, bytes] | None = None) -> None:
self.lock = Lock()
self.data = init_dict or {}
def get(self, key):
def get(self, key: str) -> bytes | None:
return self.data.get(key, None)
def set(self, key, value, expires=None):
def set(
self, key: str, value: bytes, expires: int | datetime | None = None
) -> None:
with self.lock:
self.data.update({key: value})
def delete(self, key):
def delete(self, key: str) -> None:
with self.lock:
if key in self.data:
self.data.pop(key)
@@ -55,10 +63,11 @@ class SeparateBodyBaseCache(BaseCache):
Similarly, the body should be loaded separately via ``get_body()``.
"""
def set_body(self, key, body):
def set_body(self, key: str, body: bytes) -> None:
raise NotImplementedError()
def get_body(self, key):
def get_body(self, key: str) -> IO[bytes] | None:
"""
Return the body as file-like object.
"""
@@ -2,8 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from .file_cache import FileCache, SeparateBodyFileCache
from .redis_cache import RedisCache
from pipenv.patched.pip._vendor.cachecontrol.caches.file_cache import FileCache, SeparateBodyFileCache
from pipenv.patched.pip._vendor.cachecontrol.caches.redis_cache import RedisCache
__all__ = ["FileCache", "SeparateBodyFileCache", "RedisCache"]
@@ -1,22 +1,23 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import hashlib
import os
from textwrap import dedent
from typing import IO, TYPE_CHECKING
from ..cache import BaseCache, SeparateBodyBaseCache
from ..controller import CacheController
from pipenv.patched.pip._vendor.cachecontrol.cache import BaseCache, SeparateBodyBaseCache
from pipenv.patched.pip._vendor.cachecontrol.controller import CacheController
try:
FileNotFoundError
except NameError:
# py2.X
FileNotFoundError = (IOError, OSError)
if TYPE_CHECKING:
from datetime import datetime
from filelock import BaseFileLock
def _secure_open_write(filename, fmode):
def _secure_open_write(filename: str, fmode: int) -> IO[bytes]:
# We only want to write to this file, so open it in write only mode
flags = os.O_WRONLY
@@ -39,7 +40,7 @@ def _secure_open_write(filename, fmode):
# there
try:
os.remove(filename)
except (IOError, OSError):
except OSError:
# The file must not exist already, so we can just skip ahead to opening
pass
@@ -62,37 +63,27 @@ class _FileCacheMixin:
def __init__(
self,
directory,
forever=False,
filemode=0o0600,
dirmode=0o0700,
use_dir_lock=None,
lock_class=None,
):
if use_dir_lock is not None and lock_class is not None:
raise ValueError("Cannot use use_dir_lock and lock_class together")
directory: str,
forever: bool = False,
filemode: int = 0o0600,
dirmode: int = 0o0700,
lock_class: type[BaseFileLock] | None = None,
) -> None:
try:
from lockfile import LockFile
from lockfile.mkdirlockfile import MkdirLockFile
if lock_class is None:
from filelock import FileLock
lock_class = FileLock
except ImportError:
notice = dedent(
"""
NOTE: In order to use the FileCache you must have
lockfile installed. You can install it via pip:
pip install lockfile
filelock installed. You can install it via pip:
pip install filelock
"""
)
raise ImportError(notice)
else:
if use_dir_lock:
lock_class = MkdirLockFile
elif lock_class is None:
lock_class = LockFile
self.directory = directory
self.forever = forever
self.filemode = filemode
@@ -100,17 +91,17 @@ class _FileCacheMixin:
self.lock_class = lock_class
@staticmethod
def encode(x):
def encode(x: str) -> str:
return hashlib.sha224(x.encode()).hexdigest()
def _fn(self, name):
def _fn(self, name: str) -> str:
# NOTE: This method should not change as some may depend on it.
# See: https://github.com/ionrock/cachecontrol/issues/63
hashed = self.encode(name)
parts = list(hashed[:5]) + [hashed]
return os.path.join(self.directory, *parts)
def get(self, key):
def get(self, key: str) -> bytes | None:
name = self._fn(key)
try:
with open(name, "rb") as fh:
@@ -119,26 +110,28 @@ class _FileCacheMixin:
except FileNotFoundError:
return None
def set(self, key, value, expires=None):
def set(
self, key: str, value: bytes, expires: int | datetime | None = None
) -> None:
name = self._fn(key)
self._write(name, value)
def _write(self, path, data: bytes):
def _write(self, path: str, data: bytes) -> None:
"""
Safely write the data to the given path.
"""
# Make sure the directory exists
try:
os.makedirs(os.path.dirname(path), self.dirmode)
except (IOError, OSError):
except OSError:
pass
with self.lock_class(path) as lock:
with self.lock_class(path + ".lock"):
# Write our actual file
with _secure_open_write(lock.path, self.filemode) as fh:
with _secure_open_write(path, self.filemode) as fh:
fh.write(data)
def _delete(self, key, suffix):
def _delete(self, key: str, suffix: str) -> None:
name = self._fn(key) + suffix
if not self.forever:
try:
@@ -153,7 +146,7 @@ class FileCache(_FileCacheMixin, BaseCache):
downloads.
"""
def delete(self, key):
def delete(self, key: str) -> None:
self._delete(key, "")
@@ -163,23 +156,23 @@ class SeparateBodyFileCache(_FileCacheMixin, SeparateBodyBaseCache):
peak memory usage.
"""
def get_body(self, key):
def get_body(self, key: str) -> IO[bytes] | None:
name = self._fn(key) + ".body"
try:
return open(name, "rb")
except FileNotFoundError:
return None
def set_body(self, key, body):
def set_body(self, key: str, body: bytes) -> None:
name = self._fn(key) + ".body"
self._write(name, body)
def delete(self, key):
def delete(self, key: str) -> None:
self._delete(key, "")
self._delete(key, ".body")
def url_to_file_path(url, filecache):
def url_to_file_path(url: str, filecache: FileCache) -> str:
"""Return the file cache path based on the URL.
This does not ensure the file exists!
@@ -1,39 +1,48 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from __future__ import division
from datetime import datetime
from datetime import datetime, timezone
from typing import TYPE_CHECKING
from pipenv.patched.pip._vendor.cachecontrol.cache import BaseCache
if TYPE_CHECKING:
from redis import Redis
class RedisCache(BaseCache):
def __init__(self, conn):
def __init__(self, conn: Redis[bytes]) -> None:
self.conn = conn
def get(self, key):
def get(self, key: str) -> bytes | None:
return self.conn.get(key)
def set(self, key, value, expires=None):
def set(
self, key: str, value: bytes, expires: int | datetime | None = None
) -> None:
if not expires:
self.conn.set(key, value)
elif isinstance(expires, datetime):
expires = expires - datetime.utcnow()
self.conn.setex(key, int(expires.total_seconds()), value)
now_utc = datetime.now(timezone.utc)
if expires.tzinfo is None:
now_utc = now_utc.replace(tzinfo=None)
delta = expires - now_utc
self.conn.setex(key, int(delta.total_seconds()), value)
else:
self.conn.setex(key, expires, value)
def delete(self, key):
def delete(self, key: str) -> None:
self.conn.delete(key)
def clear(self):
def clear(self) -> None:
"""Helper for clearing all the keys in a database. Use with
caution!"""
for key in self.conn.keys():
self.conn.delete(key)
def close(self):
def close(self) -> None:
"""Redis uses connection pooling, no need to close the connection."""
pass
@@ -1,32 +0,0 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
try:
from urllib.parse import urljoin
except ImportError:
from urlparse import urljoin
try:
import cPickle as pickle
except ImportError:
import pickle
# Handle the case where the requests module has been patched to not have
# urllib3 bundled as part of its source.
try:
from pipenv.patched.pip._vendor.requests.packages.urllib3.response import HTTPResponse
except ImportError:
from pipenv.patched.pip._vendor.urllib3.response import HTTPResponse
try:
from pipenv.patched.pip._vendor.requests.packages.urllib3.util import is_fp_closed
except ImportError:
from pipenv.patched.pip._vendor.urllib3.util import is_fp_closed
# Replicate some six behaviour
try:
text_type = unicode
except NameError:
text_type = str
@@ -5,17 +5,27 @@
"""
The httplib2 algorithms ported for use with requests.
"""
from __future__ import annotations
import calendar
import logging
import re
import calendar
import time
from email.utils import parsedate_tz
from typing import TYPE_CHECKING, Collection, Mapping
from pipenv.patched.pip._vendor.requests.structures import CaseInsensitiveDict
from .cache import DictCache, SeparateBodyBaseCache
from .serialize import Serializer
from pipenv.patched.pip._vendor.cachecontrol.cache import DictCache, SeparateBodyBaseCache
from pipenv.patched.pip._vendor.cachecontrol.serialize import Serializer
if TYPE_CHECKING:
from typing import Literal
from pipenv.patched.pip._vendor.requests import PreparedRequest
from pipenv.patched.pip._vendor.urllib3 import HTTPResponse
from pipenv.patched.pip._vendor.cachecontrol.cache import BaseCache
logger = logging.getLogger(__name__)
@@ -24,20 +34,26 @@ URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
PERMANENT_REDIRECT_STATUSES = (301, 308)
def parse_uri(uri):
def parse_uri(uri: str) -> tuple[str, str, str, str, str]:
"""Parses a URI using the regex given in Appendix B of RFC 3986.
(scheme, authority, path, query, fragment) = parse_uri(uri)
"""
groups = URI.match(uri).groups()
match = URI.match(uri)
assert match is not None
groups = match.groups()
return (groups[1], groups[3], groups[4], groups[6], groups[8])
class CacheController(object):
class CacheController:
"""An interface to see if request should cached or not."""
def __init__(
self, cache=None, cache_etags=True, serializer=None, status_codes=None
self,
cache: BaseCache | None = None,
cache_etags: bool = True,
serializer: Serializer | None = None,
status_codes: Collection[int] | None = None,
):
self.cache = DictCache() if cache is None else cache
self.cache_etags = cache_etags
@@ -45,7 +61,7 @@ class CacheController(object):
self.cacheable_status_codes = status_codes or (200, 203, 300, 301, 308)
@classmethod
def _urlnorm(cls, uri):
def _urlnorm(cls, uri: str) -> str:
"""Normalize the URL to create a safe key for the cache"""
(scheme, authority, path, query, fragment) = parse_uri(uri)
if not scheme or not authority:
@@ -65,10 +81,10 @@ class CacheController(object):
return defrag_uri
@classmethod
def cache_url(cls, uri):
def cache_url(cls, uri: str) -> str:
return cls._urlnorm(uri)
def parse_cache_control(self, headers):
def parse_cache_control(self, headers: Mapping[str, str]) -> dict[str, int | None]:
known_directives = {
# https://tools.ietf.org/html/rfc7234#section-5.2
"max-age": (int, True),
@@ -87,7 +103,7 @@ class CacheController(object):
cc_headers = headers.get("cache-control", headers.get("Cache-Control", ""))
retval = {}
retval: dict[str, int | None] = {}
for cc_directive in cc_headers.split(","):
if not cc_directive.strip():
@@ -122,11 +138,33 @@ class CacheController(object):
return retval
def cached_request(self, request):
def _load_from_cache(self, request: PreparedRequest) -> HTTPResponse | None:
"""
Load a cached response, or return None if it's not available.
"""
cache_url = request.url
assert cache_url is not None
cache_data = self.cache.get(cache_url)
if cache_data is None:
logger.debug("No cache entry available")
return None
if isinstance(self.cache, SeparateBodyBaseCache):
body_file = self.cache.get_body(cache_url)
else:
body_file = None
result = self.serializer.loads(request, cache_data, body_file)
if result is None:
logger.warning("Cache entry deserialization failed, entry ignored")
return result
def cached_request(self, request: PreparedRequest) -> HTTPResponse | Literal[False]:
"""
Return a cached response if it exists in the cache, otherwise
return False.
"""
assert request.url is not None
cache_url = self.cache_url(request.url)
logger.debug('Looking up "%s" in the cache', cache_url)
cc = self.parse_cache_control(request.headers)
@@ -140,21 +178,9 @@ class CacheController(object):
logger.debug('Request header has "max_age" as 0, cache bypassed')
return False
# Request allows serving from the cache, let's see if we find something
cache_data = self.cache.get(cache_url)
if cache_data is None:
logger.debug("No cache entry available")
return False
if isinstance(self.cache, SeparateBodyBaseCache):
body_file = self.cache.get_body(cache_url)
else:
body_file = None
# Check whether it can be deserialized
resp = self.serializer.loads(request, cache_data, body_file)
# Check whether we can load the response from the cache:
resp = self._load_from_cache(request)
if not resp:
logger.warning("Cache entry deserialization failed, entry ignored")
return False
# If we have a cached permanent redirect, return it immediately. We
@@ -174,7 +200,7 @@ class CacheController(object):
logger.debug(msg)
return resp
headers = CaseInsensitiveDict(resp.headers)
headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers)
if not headers or "date" not in headers:
if "etag" not in headers:
# Without date or etag, the cached response can never be used
@@ -185,7 +211,9 @@ class CacheController(object):
return False
now = time.time()
date = calendar.timegm(parsedate_tz(headers["date"]))
time_tuple = parsedate_tz(headers["date"])
assert time_tuple is not None
date = calendar.timegm(time_tuple[:6])
current_age = max(0, now - date)
logger.debug("Current age based on date: %i", current_age)
@@ -199,28 +227,30 @@ class CacheController(object):
freshness_lifetime = 0
# Check the max-age pragma in the cache control header
if "max-age" in resp_cc:
freshness_lifetime = resp_cc["max-age"]
max_age = resp_cc.get("max-age")
if max_age is not None:
freshness_lifetime = max_age
logger.debug("Freshness lifetime from max-age: %i", freshness_lifetime)
# If there isn't a max-age, check for an expires header
elif "expires" in headers:
expires = parsedate_tz(headers["expires"])
if expires is not None:
expire_time = calendar.timegm(expires) - date
expire_time = calendar.timegm(expires[:6]) - date
freshness_lifetime = max(0, expire_time)
logger.debug("Freshness lifetime from expires: %i", freshness_lifetime)
# Determine if we are setting freshness limit in the
# request. Note, this overrides what was in the response.
if "max-age" in cc:
freshness_lifetime = cc["max-age"]
max_age = cc.get("max-age")
if max_age is not None:
freshness_lifetime = max_age
logger.debug(
"Freshness lifetime from request max-age: %i", freshness_lifetime
)
if "min-fresh" in cc:
min_fresh = cc["min-fresh"]
min_fresh = cc.get("min-fresh")
if min_fresh is not None:
# adjust our current age by our min fresh
current_age += min_fresh
logger.debug("Adjusted current age from min-fresh: %i", current_age)
@@ -239,13 +269,12 @@ class CacheController(object):
# return the original handler
return False
def conditional_headers(self, request):
cache_url = self.cache_url(request.url)
resp = self.serializer.loads(request, self.cache.get(cache_url))
def conditional_headers(self, request: PreparedRequest) -> dict[str, str]:
resp = self._load_from_cache(request)
new_headers = {}
if resp:
headers = CaseInsensitiveDict(resp.headers)
headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers)
if "etag" in headers:
new_headers["If-None-Match"] = headers["ETag"]
@@ -255,7 +284,14 @@ class CacheController(object):
return new_headers
def _cache_set(self, cache_url, request, response, body=None, expires_time=None):
def _cache_set(
self,
cache_url: str,
request: PreparedRequest,
response: HTTPResponse,
body: bytes | None = None,
expires_time: int | None = None,
) -> None:
"""
Store the data in the cache.
"""
@@ -267,7 +303,10 @@ class CacheController(object):
self.serializer.dumps(request, response, b""),
expires=expires_time,
)
self.cache.set_body(cache_url, body)
# body is None can happen when, for example, we're only updating
# headers, as is the case in update_cached_response().
if body is not None:
self.cache.set_body(cache_url, body)
else:
self.cache.set(
cache_url,
@@ -275,7 +314,13 @@ class CacheController(object):
expires=expires_time,
)
def cache_response(self, request, response, body=None, status_codes=None):
def cache_response(
self,
request: PreparedRequest,
response: HTTPResponse,
body: bytes | None = None,
status_codes: Collection[int] | None = None,
) -> None:
"""
Algorithm for caching requests.
@@ -290,10 +335,14 @@ class CacheController(object):
)
return
response_headers = CaseInsensitiveDict(response.headers)
response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(
response.headers
)
if "date" in response_headers:
date = calendar.timegm(parsedate_tz(response_headers["date"]))
time_tuple = parsedate_tz(response_headers["date"])
assert time_tuple is not None
date = calendar.timegm(time_tuple[:6])
else:
date = 0
@@ -312,6 +361,7 @@ class CacheController(object):
cc_req = self.parse_cache_control(request.headers)
cc = self.parse_cache_control(response_headers)
assert request.url is not None
cache_url = self.cache_url(request.url)
logger.debug('Updating cache with response from "%s"', cache_url)
@@ -344,11 +394,11 @@ class CacheController(object):
if response_headers.get("expires"):
expires = parsedate_tz(response_headers["expires"])
if expires is not None:
expires_time = calendar.timegm(expires) - date
expires_time = calendar.timegm(expires[:6]) - date
expires_time = max(expires_time, 14 * 86400)
logger.debug("etag object cached for {0} seconds".format(expires_time))
logger.debug(f"etag object cached for {expires_time} seconds")
logger.debug("Caching due to etag")
self._cache_set(cache_url, request, response, body, expires_time)
@@ -362,11 +412,14 @@ class CacheController(object):
# is no date header then we can't do anything about expiring
# the cache.
elif "date" in response_headers:
date = calendar.timegm(parsedate_tz(response_headers["date"]))
time_tuple = parsedate_tz(response_headers["date"])
assert time_tuple is not None
date = calendar.timegm(time_tuple[:6])
# cache when there is a max-age > 0
if "max-age" in cc and cc["max-age"] > 0:
max_age = cc.get("max-age")
if max_age is not None and max_age > 0:
logger.debug("Caching b/c date exists and max-age > 0")
expires_time = cc["max-age"]
expires_time = max_age
self._cache_set(
cache_url,
request,
@@ -381,12 +434,12 @@ class CacheController(object):
if response_headers["expires"]:
expires = parsedate_tz(response_headers["expires"])
if expires is not None:
expires_time = calendar.timegm(expires) - date
expires_time = calendar.timegm(expires[:6]) - date
else:
expires_time = None
logger.debug(
"Caching b/c of expires header. expires in {0} seconds".format(
"Caching b/c of expires header. expires in {} seconds".format(
expires_time
)
)
@@ -398,16 +451,18 @@ class CacheController(object):
expires_time,
)
def update_cached_response(self, request, response):
def update_cached_response(
self, request: PreparedRequest, response: HTTPResponse
) -> HTTPResponse:
"""On a 304 we will get a new set of headers that we want to
update our cached value with, assuming we have one.
This should only ever be called when we've sent an ETag and
gotten a 304 as the response.
"""
assert request.url is not None
cache_url = self.cache_url(request.url)
cached_response = self.serializer.loads(request, self.cache.get(cache_url))
cached_response = self._load_from_cache(request)
if not cached_response:
# we didn't have a cached response
@@ -423,11 +478,11 @@ class CacheController(object):
excluded_headers = ["content-length"]
cached_response.headers.update(
dict(
(k, v)
for k, v in response.headers.items()
{
k: v
for k, v in response.headers.items() # type: ignore[no-untyped-call]
if k.lower() not in excluded_headers
)
}
)
# we want a 200 b/c we have content via the cache
@@ -1,12 +1,17 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from tempfile import NamedTemporaryFile
import mmap
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Any, Callable
if TYPE_CHECKING:
from http.client import HTTPResponse
class CallbackFileWrapper(object):
class CallbackFileWrapper:
"""
Small wrapper around a fp object which will tee everything read into a
buffer, and when that file is closed it will execute a callback with the
@@ -25,12 +30,14 @@ class CallbackFileWrapper(object):
performance impact.
"""
def __init__(self, fp, callback):
def __init__(
self, fp: HTTPResponse, callback: Callable[[bytes], None] | None
) -> None:
self.__buf = NamedTemporaryFile("rb+", delete=True)
self.__fp = fp
self.__callback = callback
def __getattr__(self, name):
def __getattr__(self, name: str) -> Any:
# The vaguaries of garbage collection means that self.__fp is
# not always set. By using __getattribute__ and the private
# name[0] allows looking up the attribute value and raising an
@@ -42,7 +49,7 @@ class CallbackFileWrapper(object):
fp = self.__getattribute__("_CallbackFileWrapper__fp")
return getattr(fp, name)
def __is_fp_closed(self):
def __is_fp_closed(self) -> bool:
try:
return self.__fp.fp is None
@@ -50,7 +57,8 @@ class CallbackFileWrapper(object):
pass
try:
return self.__fp.closed
closed: bool = self.__fp.closed
return closed
except AttributeError:
pass
@@ -59,7 +67,7 @@ class CallbackFileWrapper(object):
# TODO: Add some logging here...
return False
def _close(self):
def _close(self) -> None:
if self.__callback:
if self.__buf.tell() == 0:
# Empty file:
@@ -86,8 +94,8 @@ class CallbackFileWrapper(object):
# Important when caching big files.
self.__buf.close()
def read(self, amt=None):
data = self.__fp.read(amt)
def read(self, amt: int | None = None) -> bytes:
data: bytes = self.__fp.read(amt)
if data:
# We may be dealing with b'', a sign that things are over:
# it's passed e.g. after we've already closed self.__buf.
@@ -97,8 +105,8 @@ class CallbackFileWrapper(object):
return data
def _safe_read(self, amt):
data = self.__fp._safe_read(amt)
def _safe_read(self, amt: int) -> bytes:
data: bytes = self.__fp._safe_read(amt) # type: ignore[attr-defined]
if amt == 2 and data == b"\r\n":
# urllib executes this read to toss the CRLF at the end
# of the chunk.
@@ -1,29 +1,31 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import calendar
import time
from datetime import datetime, timedelta, timezone
from email.utils import formatdate, parsedate, parsedate_tz
from typing import TYPE_CHECKING, Any, Mapping
from datetime import datetime, timedelta
if TYPE_CHECKING:
from pipenv.patched.pip._vendor.urllib3 import HTTPResponse
TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT"
def expire_after(delta, date=None):
date = date or datetime.utcnow()
def expire_after(delta: timedelta, date: datetime | None = None) -> datetime:
date = date or datetime.now(timezone.utc)
return date + delta
def datetime_to_header(dt):
def datetime_to_header(dt: datetime) -> str:
return formatdate(calendar.timegm(dt.timetuple()))
class BaseHeuristic(object):
def warning(self, response):
class BaseHeuristic:
def warning(self, response: HTTPResponse) -> str | None:
"""
Return a valid 1xx warning header value describing the cache
adjustments.
@@ -34,7 +36,7 @@ class BaseHeuristic(object):
"""
return '110 - "Response is Stale"'
def update_headers(self, response):
def update_headers(self, response: HTTPResponse) -> dict[str, str]:
"""Update the response headers with any new headers.
NOTE: This SHOULD always include some Warning header to
@@ -43,7 +45,7 @@ class BaseHeuristic(object):
"""
return {}
def apply(self, response):
def apply(self, response: HTTPResponse) -> HTTPResponse:
updated_headers = self.update_headers(response)
if updated_headers:
@@ -61,12 +63,12 @@ class OneDayCache(BaseHeuristic):
future.
"""
def update_headers(self, response):
def update_headers(self, response: HTTPResponse) -> dict[str, str]:
headers = {}
if "expires" not in response.headers:
date = parsedate(response.headers["date"])
expires = expire_after(timedelta(days=1), date=datetime(*date[:6]))
expires = expire_after(timedelta(days=1), date=datetime(*date[:6], tzinfo=timezone.utc)) # type: ignore[misc]
headers["expires"] = datetime_to_header(expires)
headers["cache-control"] = "public"
return headers
@@ -77,14 +79,14 @@ class ExpiresAfter(BaseHeuristic):
Cache **all** requests for a defined time period.
"""
def __init__(self, **kw):
def __init__(self, **kw: Any) -> None:
self.delta = timedelta(**kw)
def update_headers(self, response):
def update_headers(self, response: HTTPResponse) -> dict[str, str]:
expires = expire_after(self.delta)
return {"expires": datetime_to_header(expires), "cache-control": "public"}
def warning(self, response):
def warning(self, response: HTTPResponse) -> str | None:
tmpl = "110 - Automatically cached for %s. Response might be stale"
return tmpl % self.delta
@@ -101,12 +103,23 @@ class LastModified(BaseHeuristic):
http://lxr.mozilla.org/mozilla-release/source/netwerk/protocol/http/nsHttpResponseHead.cpp#397
Unlike mozilla we limit this to 24-hr.
"""
cacheable_by_default_statuses = {
200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501
200,
203,
204,
206,
300,
301,
404,
405,
410,
414,
501,
}
def update_headers(self, resp):
headers = resp.headers
def update_headers(self, resp: HTTPResponse) -> dict[str, str]:
headers: Mapping[str, str] = resp.headers
if "expires" in headers:
return {}
@@ -120,9 +133,11 @@ class LastModified(BaseHeuristic):
if "date" not in headers or "last-modified" not in headers:
return {}
date = calendar.timegm(parsedate_tz(headers["date"]))
time_tuple = parsedate_tz(headers["date"])
assert time_tuple is not None
date = calendar.timegm(time_tuple[:6])
last_modified = parsedate(headers["last-modified"])
if date is None or last_modified is None:
if last_modified is None:
return {}
now = time.time()
@@ -135,5 +150,5 @@ class LastModified(BaseHeuristic):
expires = date + freshness_lifetime
return {"expires": time.strftime(TIME_FMT, time.gmtime(expires))}
def warning(self, resp):
def warning(self, resp: HTTPResponse) -> str | None:
return None
@@ -1,78 +1,76 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import base64
import io
import json
import zlib
from typing import IO, TYPE_CHECKING, Any, Mapping, cast
from pipenv.patched.pip._vendor import msgpack
from pipenv.patched.pip._vendor.requests.structures import CaseInsensitiveDict
from pipenv.patched.pip._vendor.urllib3 import HTTPResponse
from .compat import HTTPResponse, pickle, text_type
if TYPE_CHECKING:
from pipenv.patched.pip._vendor.requests import PreparedRequest
def _b64_decode_bytes(b):
return base64.b64decode(b.encode("ascii"))
class Serializer:
serde_version = "4"
def _b64_decode_str(s):
return _b64_decode_bytes(s).decode("utf8")
_default_body_read = object()
class Serializer(object):
def dumps(self, request, response, body=None):
response_headers = CaseInsensitiveDict(response.headers)
def dumps(
self,
request: PreparedRequest,
response: HTTPResponse,
body: bytes | None = None,
) -> bytes:
response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(
response.headers
)
if body is None:
# When a body isn't passed in, we'll read the response. We
# also update the response with a new file handler to be
# sure it acts as though it was never read.
body = response.read(decode_content=False)
response._fp = io.BytesIO(body)
response._fp = io.BytesIO(body) # type: ignore[attr-defined]
response.length_remaining = len(body)
# NOTE: This is all a bit weird, but it's really important that on
# Python 2.x these objects are unicode and not str, even when
# they contain only ascii. The problem here is that msgpack
# understands the difference between unicode and bytes and we
# have it set to differentiate between them, however Python 2
# doesn't know the difference. Forcing these to unicode will be
# enough to have msgpack know the difference.
data = {
u"response": {
u"body": body, # Empty bytestring if body is stored separately
u"headers": dict(
(text_type(k), text_type(v)) for k, v in response.headers.items()
),
u"status": response.status,
u"version": response.version,
u"reason": text_type(response.reason),
u"strict": response.strict,
u"decode_content": response.decode_content,
"response": {
"body": body, # Empty bytestring if body is stored separately
"headers": {str(k): str(v) for k, v in response.headers.items()}, # type: ignore[no-untyped-call]
"status": response.status,
"version": response.version,
"reason": str(response.reason),
"decode_content": response.decode_content,
}
}
# Construct our vary headers
data[u"vary"] = {}
if u"vary" in response_headers:
varied_headers = response_headers[u"vary"].split(",")
data["vary"] = {}
if "vary" in response_headers:
varied_headers = response_headers["vary"].split(",")
for header in varied_headers:
header = text_type(header).strip()
header = str(header).strip()
header_value = request.headers.get(header, None)
if header_value is not None:
header_value = text_type(header_value)
data[u"vary"][header] = header_value
header_value = str(header_value)
data["vary"][header] = header_value
return b",".join([b"cc=4", msgpack.dumps(data, use_bin_type=True)])
return b",".join([f"cc={self.serde_version}".encode(), self.serialize(data)])
def loads(self, request, data, body_file=None):
def serialize(self, data: dict[str, Any]) -> bytes:
return cast(bytes, msgpack.dumps(data, use_bin_type=True))
def loads(
self,
request: PreparedRequest,
data: bytes,
body_file: IO[bytes] | None = None,
) -> HTTPResponse | None:
# Short circuit if we've been given an empty set of data
if not data:
return
return None
# Determine what version of the serializer the data was serialized
# with
@@ -88,18 +86,23 @@ class Serializer(object):
ver = b"cc=0"
# Get the version number out of the cc=N
ver = ver.split(b"=", 1)[-1].decode("ascii")
verstr = ver.split(b"=", 1)[-1].decode("ascii")
# Dispatch to the actual load method for the given version
try:
return getattr(self, "_loads_v{}".format(ver))(request, data, body_file)
return getattr(self, f"_loads_v{verstr}")(request, data, body_file) # type: ignore[no-any-return]
except AttributeError:
# This is a version we don't have a loads function for, so we'll
# just treat it as a miss and return None
return
return None
def prepare_response(self, request, cached, body_file=None):
def prepare_response(
self,
request: PreparedRequest,
cached: Mapping[str, Any],
body_file: IO[bytes] | None = None,
) -> HTTPResponse | None:
"""Verify our vary headers match and construct a real urllib3
HTTPResponse object.
"""
@@ -108,23 +111,26 @@ class Serializer(object):
# This case is also handled in the controller code when creating
# a cache entry, but is left here for backwards compatibility.
if "*" in cached.get("vary", {}):
return
return None
# Ensure that the Vary headers for the cached response match our
# request
for header, value in cached.get("vary", {}).items():
if request.headers.get(header, None) != value:
return
return None
body_raw = cached["response"].pop("body")
headers = CaseInsensitiveDict(data=cached["response"]["headers"])
headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(
data=cached["response"]["headers"]
)
if headers.get("transfer-encoding", "") == "chunked":
headers.pop("transfer-encoding")
cached["response"]["headers"] = headers
try:
body: IO[bytes]
if body_file is None:
body = io.BytesIO(body_raw)
else:
@@ -138,53 +144,63 @@ class Serializer(object):
# TypeError: 'str' does not support the buffer interface
body = io.BytesIO(body_raw.encode("utf8"))
# Discard any `strict` parameter serialized by older version of cachecontrol.
cached["response"].pop("strict", None)
return HTTPResponse(body=body, preload_content=False, **cached["response"])
def _loads_v0(self, request, data, body_file=None):
def _loads_v0(
self,
request: PreparedRequest,
data: bytes,
body_file: IO[bytes] | None = None,
) -> None:
# The original legacy cache data. This doesn't contain enough
# information to construct everything we need, so we'll treat this as
# a miss.
return
return None
def _loads_v1(self, request, data, body_file=None):
try:
cached = pickle.loads(data)
except ValueError:
return
def _loads_v1(
self,
request: PreparedRequest,
data: bytes,
body_file: IO[bytes] | None = None,
) -> HTTPResponse | None:
# The "v1" pickled cache format. This is no longer supported
# for security reasons, so we treat it as a miss.
return None
return self.prepare_response(request, cached, body_file)
def _loads_v2(
self,
request: PreparedRequest,
data: bytes,
body_file: IO[bytes] | None = None,
) -> HTTPResponse | None:
# The "v2" compressed base64 cache format.
# This has been removed due to age and poor size/performance
# characteristics, so we treat it as a miss.
return None
def _loads_v2(self, request, data, body_file=None):
assert body_file is None
try:
cached = json.loads(zlib.decompress(data).decode("utf8"))
except (ValueError, zlib.error):
return
# We need to decode the items that we've base64 encoded
cached["response"]["body"] = _b64_decode_bytes(cached["response"]["body"])
cached["response"]["headers"] = dict(
(_b64_decode_str(k), _b64_decode_str(v))
for k, v in cached["response"]["headers"].items()
)
cached["response"]["reason"] = _b64_decode_str(cached["response"]["reason"])
cached["vary"] = dict(
(_b64_decode_str(k), _b64_decode_str(v) if v is not None else v)
for k, v in cached["vary"].items()
)
return self.prepare_response(request, cached, body_file)
def _loads_v3(self, request, data, body_file):
def _loads_v3(
self,
request: PreparedRequest,
data: bytes,
body_file: IO[bytes] | None = None,
) -> None:
# Due to Python 2 encoding issues, it's impossible to know for sure
# exactly how to load v3 entries, thus we'll treat these as a miss so
# that they get rewritten out as v4 entries.
return
return None
def _loads_v4(self, request, data, body_file=None):
def _loads_v4(
self,
request: PreparedRequest,
data: bytes,
body_file: IO[bytes] | None = None,
) -> HTTPResponse | None:
try:
cached = msgpack.loads(data, raw=False)
except ValueError:
return
return None
return self.prepare_response(request, cached, body_file)
@@ -1,22 +1,32 @@
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from .adapter import CacheControlAdapter
from .cache import DictCache
from typing import TYPE_CHECKING, Collection
from pipenv.patched.pip._vendor.cachecontrol.adapter import CacheControlAdapter
from pipenv.patched.pip._vendor.cachecontrol.cache import DictCache
if TYPE_CHECKING:
from pipenv.patched.pip._vendor import requests
from pipenv.patched.pip._vendor.cachecontrol.cache import BaseCache
from pipenv.patched.pip._vendor.cachecontrol.controller import CacheController
from pipenv.patched.pip._vendor.cachecontrol.heuristics import BaseHeuristic
from pipenv.patched.pip._vendor.cachecontrol.serialize import Serializer
def CacheControl(
sess,
cache=None,
cache_etags=True,
serializer=None,
heuristic=None,
controller_class=None,
adapter_class=None,
cacheable_methods=None,
):
sess: requests.Session,
cache: BaseCache | None = None,
cache_etags: bool = True,
serializer: Serializer | None = None,
heuristic: BaseHeuristic | None = None,
controller_class: type[CacheController] | None = None,
adapter_class: type[CacheControlAdapter] | None = None,
cacheable_methods: Collection[str] | None = None,
) -> requests.Session:
cache = DictCache() if cache is None else cache
adapter_class = adapter_class or CacheControlAdapter
adapter = adapter_class(
@@ -1,4 +1,4 @@
from .core import contents, where
__all__ = ["contents", "where"]
__version__ = "2023.05.07"
__version__ = "2023.07.22"
+185 -30
View File
@@ -791,34 +791,6 @@ uLjbvrW5KfnaNwUASZQDhETnv0Mxz3WLJdH0pmT1kvarBes96aULNmLazAZfNou2
XjG4Kvte9nHfRCaexOYNkbQudZWAUWpLMKawYqGT8ZvYzsRjdT9ZR7E=
-----END CERTIFICATE-----
# Issuer: CN=Hongkong Post Root CA 1 O=Hongkong Post
# Subject: CN=Hongkong Post Root CA 1 O=Hongkong Post
# Label: "Hongkong Post Root CA 1"
# Serial: 1000
# MD5 Fingerprint: a8:0d:6f:39:78:b9:43:6d:77:42:6d:98:5a:cc:23:ca
# SHA1 Fingerprint: d6:da:a8:20:8d:09:d2:15:4d:24:b5:2f:cb:34:6e:b2:58:b2:8a:58
# SHA256 Fingerprint: f9:e6:7d:33:6c:51:00:2a:c0:54:c6:32:02:2d:66:dd:a2:e7:e3:ff:f1:0a:d0:61:ed:31:d8:bb:b4:10:cf:b2
-----BEGIN CERTIFICATE-----
MIIDMDCCAhigAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwRzELMAkGA1UEBhMCSEsx
FjAUBgNVBAoTDUhvbmdrb25nIFBvc3QxIDAeBgNVBAMTF0hvbmdrb25nIFBvc3Qg
Um9vdCBDQSAxMB4XDTAzMDUxNTA1MTMxNFoXDTIzMDUxNTA0NTIyOVowRzELMAkG
A1UEBhMCSEsxFjAUBgNVBAoTDUhvbmdrb25nIFBvc3QxIDAeBgNVBAMTF0hvbmdr
b25nIFBvc3QgUm9vdCBDQSAxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
AQEArP84tulmAknjorThkPlAj3n54r15/gK97iSSHSL22oVyaf7XPwnU3ZG1ApzQ
jVrhVcNQhrkpJsLj2aDxaQMoIIBFIi1WpztUlVYiWR8o3x8gPW2iNr4joLFutbEn
PzlTCeqrauh0ssJlXI6/fMN4hM2eFvz1Lk8gKgifd/PFHsSaUmYeSF7jEAaPIpjh
ZY4bXSNmO7ilMlHIhqqhqZ5/dpTCpmy3QfDVyAY45tQM4vM7TG1QjMSDJ8EThFk9
nnV0ttgCXjqQesBCNnLsak3c78QA3xMYV18meMjWCnl3v/evt3a5pQuEF10Q6m/h
q5URX208o1xNg1vysxmKgIsLhwIDAQABoyYwJDASBgNVHRMBAf8ECDAGAQH/AgED
MA4GA1UdDwEB/wQEAwIBxjANBgkqhkiG9w0BAQUFAAOCAQEADkbVPK7ih9legYsC
mEEIjEy82tvuJxuC52pF7BaLT4Wg87JwvVqWuspube5Gi27nKi6Wsxkz67SfqLI3
7piol7Yutmcn1KZJ/RyTZXaeQi/cImyaT/JaFTmxcdcrUehtHJjA2Sr0oYJ71clB
oiMBdDhViw+5LmeiIAQ32pwL0xch4I+XeTRvhEgCIDMb5jREn5Fw9IBehEPCKdJs
EhTkYY2sEJCehFC78JZvRZ+K88psT/oROhUVRsPNH4NbLUES7VBnQRM9IauUiqpO
fMGx+6fWtScvl6tu4B3i0RwsH0Ti/L6RoZz71ilTc4afU9hDDl3WY4JxHYB0yvbi
AmvZWg==
-----END CERTIFICATE-----
# Issuer: CN=SecureSign RootCA11 O=Japan Certification Services, Inc.
# Subject: CN=SecureSign RootCA11 O=Japan Certification Services, Inc.
# Label: "SecureSign RootCA11"
@@ -1676,7 +1648,6 @@ HL/EVlP6Y2XQ8xwOFvVrhlhNGNTkDY6lnVuR3HYkUD/GKvvZt5y11ubQ2egZixVx
SK236thZiNSQvxaz2emsWWFUyBy6ysHK4bkgTI86k4mloMy/0/Z1pHWWbVY=
-----END CERTIFICATE-----
# Issuer: CN=T-TeleSec GlobalRoot Class 2 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center
# Subject: CN=T-TeleSec GlobalRoot Class 2 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center
# Label: "T-TeleSec GlobalRoot Class 2"
@@ -4354,7 +4325,6 @@ ut6Dacpps6kFtZaSF4fC0urQe87YQVt8rgIwRt7qy12a7DLCZRawTDBcMPPaTnOG
BtjOiQRINzf43TNRnXCve1XYAS59BWQOhriR
-----END CERTIFICATE-----
# Issuer: CN=Security Communication RootCA3 O=SECOM Trust Systems CO.,LTD.
# Subject: CN=Security Communication RootCA3 O=SECOM Trust Systems CO.,LTD.
# Label: "Security Communication RootCA3"
@@ -4478,3 +4448,188 @@ AgEGMAoGCCqGSM49BAMDA2gAMGUCMBq8W9f+qdJUDkpd0m2xQNz0Q9XSSpkZElaA
94M04TVOSG0ED1cxMDAtsaqdAzjbBgIxAMvMh1PLet8gUXOQwKhbYdDFUDn9hf7B
43j4ptZLvZuHjw/l1lOWqzzIQNph91Oj9w==
-----END CERTIFICATE-----
# Issuer: CN=Sectigo Public Server Authentication Root E46 O=Sectigo Limited
# Subject: CN=Sectigo Public Server Authentication Root E46 O=Sectigo Limited
# Label: "Sectigo Public Server Authentication Root E46"
# Serial: 88989738453351742415770396670917916916
# MD5 Fingerprint: 28:23:f8:b2:98:5c:37:16:3b:3e:46:13:4e:b0:b3:01
# SHA1 Fingerprint: ec:8a:39:6c:40:f0:2e:bc:42:75:d4:9f:ab:1c:1a:5b:67:be:d2:9a
# SHA256 Fingerprint: c9:0f:26:f0:fb:1b:40:18:b2:22:27:51:9b:5c:a2:b5:3e:2c:a5:b3:be:5c:f1:8e:fe:1b:ef:47:38:0c:53:83
-----BEGIN CERTIFICATE-----
MIICOjCCAcGgAwIBAgIQQvLM2htpN0RfFf51KBC49DAKBggqhkjOPQQDAzBfMQsw
CQYDVQQGEwJHQjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1T
ZWN0aWdvIFB1YmxpYyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBFNDYwHhcN
MjEwMzIyMDAwMDAwWhcNNDYwMzIxMjM1OTU5WjBfMQswCQYDVQQGEwJHQjEYMBYG
A1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1TZWN0aWdvIFB1YmxpYyBT
ZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBFNDYwdjAQBgcqhkjOPQIBBgUrgQQA
IgNiAAR2+pmpbiDt+dd34wc7qNs9Xzjoq1WmVk/WSOrsfy2qw7LFeeyZYX8QeccC
WvkEN/U0NSt3zn8gj1KjAIns1aeibVvjS5KToID1AZTc8GgHHs3u/iVStSBDHBv+
6xnOQ6OjQjBAMB0GA1UdDgQWBBTRItpMWfFLXyY4qp3W7usNw/upYTAOBgNVHQ8B
Af8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNnADBkAjAn7qRa
qCG76UeXlImldCBteU/IvZNeWBj7LRoAasm4PdCkT0RHlAFWovgzJQxC36oCMB3q
4S6ILuH5px0CMk7yn2xVdOOurvulGu7t0vzCAxHrRVxgED1cf5kDW21USAGKcw==
-----END CERTIFICATE-----
# Issuer: CN=Sectigo Public Server Authentication Root R46 O=Sectigo Limited
# Subject: CN=Sectigo Public Server Authentication Root R46 O=Sectigo Limited
# Label: "Sectigo Public Server Authentication Root R46"
# Serial: 156256931880233212765902055439220583700
# MD5 Fingerprint: 32:10:09:52:00:d5:7e:6c:43:df:15:c0:b1:16:93:e5
# SHA1 Fingerprint: ad:98:f9:f3:e4:7d:75:3b:65:d4:82:b3:a4:52:17:bb:6e:f5:e4:38
# SHA256 Fingerprint: 7b:b6:47:a6:2a:ee:ac:88:bf:25:7a:a5:22:d0:1f:fe:a3:95:e0:ab:45:c7:3f:93:f6:56:54:ec:38:f2:5a:06
-----BEGIN CERTIFICATE-----
MIIFijCCA3KgAwIBAgIQdY39i658BwD6qSWn4cetFDANBgkqhkiG9w0BAQwFADBf
MQswCQYDVQQGEwJHQjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQD
Ey1TZWN0aWdvIFB1YmxpYyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBSNDYw
HhcNMjEwMzIyMDAwMDAwWhcNNDYwMzIxMjM1OTU5WjBfMQswCQYDVQQGEwJHQjEY
MBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1TZWN0aWdvIFB1Ymxp
YyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBSNDYwggIiMA0GCSqGSIb3DQEB
AQUAA4ICDwAwggIKAoICAQCTvtU2UnXYASOgHEdCSe5jtrch/cSV1UgrJnwUUxDa
ef0rty2k1Cz66jLdScK5vQ9IPXtamFSvnl0xdE8H/FAh3aTPaE8bEmNtJZlMKpnz
SDBh+oF8HqcIStw+KxwfGExxqjWMrfhu6DtK2eWUAtaJhBOqbchPM8xQljeSM9xf
iOefVNlI8JhD1mb9nxc4Q8UBUQvX4yMPFF1bFOdLvt30yNoDN9HWOaEhUTCDsG3X
ME6WW5HwcCSrv0WBZEMNvSE6Lzzpng3LILVCJ8zab5vuZDCQOc2TZYEhMbUjUDM3
IuM47fgxMMxF/mL50V0yeUKH32rMVhlATc6qu/m1dkmU8Sf4kaWD5QazYw6A3OAS
VYCmO2a0OYctyPDQ0RTp5A1NDvZdV3LFOxxHVp3i1fuBYYzMTYCQNFu31xR13NgE
SJ/AwSiItOkcyqex8Va3e0lMWeUgFaiEAin6OJRpmkkGj80feRQXEgyDet4fsZfu
+Zd4KKTIRJLpfSYFplhym3kT2BFfrsU4YjRosoYwjviQYZ4ybPUHNs2iTG7sijbt
8uaZFURww3y8nDnAtOFr94MlI1fZEoDlSfB1D++N6xybVCi0ITz8fAr/73trdf+L
HaAZBav6+CuBQug4urv7qv094PPK306Xlynt8xhW6aWWrL3DkJiy4Pmi1KZHQ3xt
zwIDAQABo0IwQDAdBgNVHQ4EFgQUVnNYZJX5khqwEioEYnmhQBWIIUkwDgYDVR0P
AQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAC9c
mTz8Bl6MlC5w6tIyMY208FHVvArzZJ8HXtXBc2hkeqK5Duj5XYUtqDdFqij0lgVQ
YKlJfp/imTYpE0RHap1VIDzYm/EDMrraQKFz6oOht0SmDpkBm+S8f74TlH7Kph52
gDY9hAaLMyZlbcp+nv4fjFg4exqDsQ+8FxG75gbMY/qB8oFM2gsQa6H61SilzwZA
Fv97fRheORKkU55+MkIQpiGRqRxOF3yEvJ+M0ejf5lG5Nkc/kLnHvALcWxxPDkjB
JYOcCj+esQMzEhonrPcibCTRAUH4WAP+JWgiH5paPHxsnnVI84HxZmduTILA7rpX
DhjvLpr3Etiga+kFpaHpaPi8TD8SHkXoUsCjvxInebnMMTzD9joiFgOgyY9mpFui
TdaBJQbpdqQACj7LzTWb4OE4y2BThihCQRxEV+ioratF4yUQvNs+ZUH7G6aXD+u5
dHn5HrwdVw1Hr8Mvn4dGp+smWg9WY7ViYG4A++MnESLn/pmPNPW56MORcr3Ywx65
LvKRRFHQV80MNNVIIb/bE/FmJUNS0nAiNs2fxBx1IK1jcmMGDw4nztJqDby1ORrp
0XZ60Vzk50lJLVU3aPAaOpg+VBeHVOmmJ1CJeyAvP/+/oYtKR5j/K3tJPsMpRmAY
QqszKbrAKbkTidOIijlBO8n9pu0f9GBj39ItVQGL
-----END CERTIFICATE-----
# Issuer: CN=SSL.com TLS RSA Root CA 2022 O=SSL Corporation
# Subject: CN=SSL.com TLS RSA Root CA 2022 O=SSL Corporation
# Label: "SSL.com TLS RSA Root CA 2022"
# Serial: 148535279242832292258835760425842727825
# MD5 Fingerprint: d8:4e:c6:59:30:d8:fe:a0:d6:7a:5a:2c:2c:69:78:da
# SHA1 Fingerprint: ec:2c:83:40:72:af:26:95:10:ff:0e:f2:03:ee:31:70:f6:78:9d:ca
# SHA256 Fingerprint: 8f:af:7d:2e:2c:b4:70:9b:b8:e0:b3:36:66:bf:75:a5:dd:45:b5:de:48:0f:8e:a8:d4:bf:e6:be:bc:17:f2:ed
-----BEGIN CERTIFICATE-----
MIIFiTCCA3GgAwIBAgIQb77arXO9CEDii02+1PdbkTANBgkqhkiG9w0BAQsFADBO
MQswCQYDVQQGEwJVUzEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMSUwIwYDVQQD
DBxTU0wuY29tIFRMUyBSU0EgUm9vdCBDQSAyMDIyMB4XDTIyMDgyNTE2MzQyMloX
DTQ2MDgxOTE2MzQyMVowTjELMAkGA1UEBhMCVVMxGDAWBgNVBAoMD1NTTCBDb3Jw
b3JhdGlvbjElMCMGA1UEAwwcU1NMLmNvbSBUTFMgUlNBIFJvb3QgQ0EgMjAyMjCC
AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANCkCXJPQIgSYT41I57u9nTP
L3tYPc48DRAokC+X94xI2KDYJbFMsBFMF3NQ0CJKY7uB0ylu1bUJPiYYf7ISf5OY
t6/wNr/y7hienDtSxUcZXXTzZGbVXcdotL8bHAajvI9AI7YexoS9UcQbOcGV0ins
S657Lb85/bRi3pZ7QcacoOAGcvvwB5cJOYF0r/c0WRFXCsJbwST0MXMwgsadugL3
PnxEX4MN8/HdIGkWCVDi1FW24IBydm5MR7d1VVm0U3TZlMZBrViKMWYPHqIbKUBO
L9975hYsLfy/7PO0+r4Y9ptJ1O4Fbtk085zx7AGL0SDGD6C1vBdOSHtRwvzpXGk3
R2azaPgVKPC506QVzFpPulJwoxJF3ca6TvvC0PeoUidtbnm1jPx7jMEWTO6Af77w
dr5BUxIzrlo4QqvXDz5BjXYHMtWrifZOZ9mxQnUjbvPNQrL8VfVThxc7wDNY8VLS
+YCk8OjwO4s4zKTGkH8PnP2L0aPP2oOnaclQNtVcBdIKQXTbYxE3waWglksejBYS
d66UNHsef8JmAOSqg+qKkK3ONkRN0VHpvB/zagX9wHQfJRlAUW7qglFA35u5CCoG
AtUjHBPW6dvbxrB6y3snm/vg1UYk7RBLY0ulBY+6uB0rpvqR4pJSvezrZ5dtmi2f
gTIFZzL7SAg/2SW4BCUvAgMBAAGjYzBhMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0j
BBgwFoAU+y437uOEeicuzRk1sTN8/9REQrkwHQYDVR0OBBYEFPsuN+7jhHonLs0Z
NbEzfP/UREK5MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAjYlt
hEUY8U+zoO9opMAdrDC8Z2awms22qyIZZtM7QbUQnRC6cm4pJCAcAZli05bg4vsM
QtfhWsSWTVTNj8pDU/0quOr4ZcoBwq1gaAafORpR2eCNJvkLTqVTJXojpBzOCBvf
R4iyrT7gJ4eLSYwfqUdYe5byiB0YrrPRpgqU+tvT5TgKa3kSM/tKWTcWQA673vWJ
DPFs0/dRa1419dvAJuoSc06pkZCmF8NsLzjUo3KUQyxi4U5cMj29TH0ZR6LDSeeW
P4+a0zvkEdiLA9z2tmBVGKaBUfPhqBVq6+AL8BQx1rmMRTqoENjwuSfr98t67wVy
lrXEj5ZzxOhWc5y8aVFjvO9nHEMaX3cZHxj4HCUp+UmZKbaSPaKDN7EgkaibMOlq
bLQjk2UEqxHzDh1TJElTHaE/nUiSEeJ9DU/1172iWD54nR4fK/4huxoTtrEoZP2w
AgDHbICivRZQIA9ygV/MlP+7mea6kMvq+cYMwq7FGc4zoWtcu358NFcXrfA/rs3q
r5nsLFR+jM4uElZI7xc7P0peYNLcdDa8pUNjyw9bowJWCZ4kLOGGgYz+qxcs+sji
Mho6/4UIyYOf8kpIEFR3N+2ivEC+5BB09+Rbu7nzifmPQdjH5FCQNYA+HLhNkNPU
98OwoX6EyneSMSy4kLGCenROmxMmtNVQZlR4rmA=
-----END CERTIFICATE-----
# Issuer: CN=SSL.com TLS ECC Root CA 2022 O=SSL Corporation
# Subject: CN=SSL.com TLS ECC Root CA 2022 O=SSL Corporation
# Label: "SSL.com TLS ECC Root CA 2022"
# Serial: 26605119622390491762507526719404364228
# MD5 Fingerprint: 99:d7:5c:f1:51:36:cc:e9:ce:d9:19:2e:77:71:56:c5
# SHA1 Fingerprint: 9f:5f:d9:1a:54:6d:f5:0c:71:f0:ee:7a:bd:17:49:98:84:73:e2:39
# SHA256 Fingerprint: c3:2f:fd:9f:46:f9:36:d1:6c:36:73:99:09:59:43:4b:9a:d6:0a:af:bb:9e:7c:f3:36:54:f1:44:cc:1b:a1:43
-----BEGIN CERTIFICATE-----
MIICOjCCAcCgAwIBAgIQFAP1q/s3ixdAW+JDsqXRxDAKBggqhkjOPQQDAzBOMQsw
CQYDVQQGEwJVUzEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMSUwIwYDVQQDDBxT
U0wuY29tIFRMUyBFQ0MgUm9vdCBDQSAyMDIyMB4XDTIyMDgyNTE2MzM0OFoXDTQ2
MDgxOTE2MzM0N1owTjELMAkGA1UEBhMCVVMxGDAWBgNVBAoMD1NTTCBDb3Jwb3Jh
dGlvbjElMCMGA1UEAwwcU1NMLmNvbSBUTFMgRUNDIFJvb3QgQ0EgMjAyMjB2MBAG
ByqGSM49AgEGBSuBBAAiA2IABEUpNXP6wrgjzhR9qLFNoFs27iosU8NgCTWyJGYm
acCzldZdkkAZDsalE3D07xJRKF3nzL35PIXBz5SQySvOkkJYWWf9lCcQZIxPBLFN
SeR7T5v15wj4A4j3p8OSSxlUgaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSME
GDAWgBSJjy+j6CugFFR781a4Jl9nOAuc0DAdBgNVHQ4EFgQUiY8vo+groBRUe/NW
uCZfZzgLnNAwDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMDA2gAMGUCMFXjIlbp
15IkWE8elDIPDAI2wv2sdDJO4fscgIijzPvX6yv/N33w7deedWo1dlJF4AIxAMeN
b0Igj762TVntd00pxCAgRWSGOlDGxK0tk/UYfXLtqc/ErFc2KAhl3zx5Zn6g6g==
-----END CERTIFICATE-----
# Issuer: CN=Atos TrustedRoot Root CA ECC TLS 2021 O=Atos
# Subject: CN=Atos TrustedRoot Root CA ECC TLS 2021 O=Atos
# Label: "Atos TrustedRoot Root CA ECC TLS 2021"
# Serial: 81873346711060652204712539181482831616
# MD5 Fingerprint: 16:9f:ad:f1:70:ad:79:d6:ed:29:b4:d1:c5:79:70:a8
# SHA1 Fingerprint: 9e:bc:75:10:42:b3:02:f3:81:f4:f7:30:62:d4:8f:c3:a7:51:b2:dd
# SHA256 Fingerprint: b2:fa:e5:3e:14:cc:d7:ab:92:12:06:47:01:ae:27:9c:1d:89:88:fa:cb:77:5f:a8:a0:08:91:4e:66:39:88:a8
-----BEGIN CERTIFICATE-----
MIICFTCCAZugAwIBAgIQPZg7pmY9kGP3fiZXOATvADAKBggqhkjOPQQDAzBMMS4w
LAYDVQQDDCVBdG9zIFRydXN0ZWRSb290IFJvb3QgQ0EgRUNDIFRMUyAyMDIxMQ0w
CwYDVQQKDARBdG9zMQswCQYDVQQGEwJERTAeFw0yMTA0MjIwOTI2MjNaFw00MTA0
MTcwOTI2MjJaMEwxLjAsBgNVBAMMJUF0b3MgVHJ1c3RlZFJvb3QgUm9vdCBDQSBF
Q0MgVExTIDIwMjExDTALBgNVBAoMBEF0b3MxCzAJBgNVBAYTAkRFMHYwEAYHKoZI
zj0CAQYFK4EEACIDYgAEloZYKDcKZ9Cg3iQZGeHkBQcfl+3oZIK59sRxUM6KDP/X
tXa7oWyTbIOiaG6l2b4siJVBzV3dscqDY4PMwL502eCdpO5KTlbgmClBk1IQ1SQ4
AjJn8ZQSb+/Xxd4u/RmAo0IwQDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR2
KCXWfeBmmnoJsmo7jjPXNtNPojAOBgNVHQ8BAf8EBAMCAYYwCgYIKoZIzj0EAwMD
aAAwZQIwW5kp85wxtolrbNa9d+F851F+uDrNozZffPc8dz7kUK2o59JZDCaOMDtu
CCrCp1rIAjEAmeMM56PDr9NJLkaCI2ZdyQAUEv049OGYa3cpetskz2VAv9LcjBHo
9H1/IISpQuQo
-----END CERTIFICATE-----
# Issuer: CN=Atos TrustedRoot Root CA RSA TLS 2021 O=Atos
# Subject: CN=Atos TrustedRoot Root CA RSA TLS 2021 O=Atos
# Label: "Atos TrustedRoot Root CA RSA TLS 2021"
# Serial: 111436099570196163832749341232207667876
# MD5 Fingerprint: d4:d3:46:b8:9a:c0:9c:76:5d:9e:3a:c3:b9:99:31:d2
# SHA1 Fingerprint: 18:52:3b:0d:06:37:e4:d6:3a:df:23:e4:98:fb:5b:16:fb:86:74:48
# SHA256 Fingerprint: 81:a9:08:8e:a5:9f:b3:64:c5:48:a6:f8:55:59:09:9b:6f:04:05:ef:bf:18:e5:32:4e:c9:f4:57:ba:00:11:2f
-----BEGIN CERTIFICATE-----
MIIFZDCCA0ygAwIBAgIQU9XP5hmTC/srBRLYwiqipDANBgkqhkiG9w0BAQwFADBM
MS4wLAYDVQQDDCVBdG9zIFRydXN0ZWRSb290IFJvb3QgQ0EgUlNBIFRMUyAyMDIx
MQ0wCwYDVQQKDARBdG9zMQswCQYDVQQGEwJERTAeFw0yMTA0MjIwOTIxMTBaFw00
MTA0MTcwOTIxMDlaMEwxLjAsBgNVBAMMJUF0b3MgVHJ1c3RlZFJvb3QgUm9vdCBD
QSBSU0EgVExTIDIwMjExDTALBgNVBAoMBEF0b3MxCzAJBgNVBAYTAkRFMIICIjAN
BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtoAOxHm9BYx9sKOdTSJNy/BBl01Z
4NH+VoyX8te9j2y3I49f1cTYQcvyAh5x5en2XssIKl4w8i1mx4QbZFc4nXUtVsYv
Ye+W/CBGvevUez8/fEc4BKkbqlLfEzfTFRVOvV98r61jx3ncCHvVoOX3W3WsgFWZ
kmGbzSoXfduP9LVq6hdKZChmFSlsAvFr1bqjM9xaZ6cF4r9lthawEO3NUDPJcFDs
GY6wx/J0W2tExn2WuZgIWWbeKQGb9Cpt0xU6kGpn8bRrZtkh68rZYnxGEFzedUln
nkL5/nWpo63/dgpnQOPF943HhZpZnmKaau1Fh5hnstVKPNe0OwANwI8f4UDErmwh
3El+fsqyjW22v5MvoVw+j8rtgI5Y4dtXz4U2OLJxpAmMkokIiEjxQGMYsluMWuPD
0xeqqxmjLBvk1cbiZnrXghmmOxYsL3GHX0WelXOTwkKBIROW1527k2gV+p2kHYzy
geBYBr3JtuP2iV2J+axEoctr+hbxx1A9JNr3w+SH1VbxT5Aw+kUJWdo0zuATHAR8
ANSbhqRAvNncTFd+rrcztl524WWLZt+NyteYr842mIycg5kDcPOvdO3GDjbnvezB
c6eUWsuSZIKmAMFwoW4sKeFYV+xafJlrJaSQOoD0IJ2azsct+bJLKZWD6TWNp0lI
pw9MGZHQ9b8Q4HECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU
dEmZ0f+0emhFdcN+tNzMzjkz2ggwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB
DAUAA4ICAQAjQ1MkYlxt/T7Cz1UAbMVWiLkO3TriJQ2VSpfKgInuKs1l+NsW4AmS
4BjHeJi78+xCUvuppILXTdiK/ORO/auQxDh1MoSf/7OwKwIzNsAQkG8dnK/haZPs
o0UvFJ/1TCplQ3IM98P4lYsU84UgYt1UU90s3BiVaU+DR3BAM1h3Egyi61IxHkzJ
qM7F78PRreBrAwA0JrRUITWXAdxfG/F851X6LWh3e9NpzNMOa7pNdkTWwhWaJuyw
xfW70Xp0wmzNxbVe9kzmWy2B27O3Opee7c9GslA9hGCZcbUztVdF5kJHdWoOsAgM
rr3e97sPWD2PAzHoPYJQyi9eDF20l74gNAf0xBLh7tew2VktafcxBPTy+av5EzH4
AXcOPUIjJsyacmdRIXrMPIWo6iFqO9taPKU0nprALN+AnCng33eU0aKAQv9qTFsR
0PXNor6uzFFcw9VUewyu1rkGd4Di7wcaaMxZUa1+XGdrudviB0JbuAEFWDlN5LuY
o7Ey7Nmj1m+UI/87tyll5gfp77YZ6ufCOB0yiJA8EytuzO+rdwY0d4RPcuSBhPm5
dDTedk+SKlOxJTnbPP/lPqYO5Wue/9vsL3SD3460s6neFE3/MaNFcyT6lSnMEpcE
oji2jbDwN/zIIX8/syQbPYtuzE2wFg2WHYMfRsCbvUOZ58SWLs5fyQ==
-----END CERTIFICATE-----
@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2022 Seth Michael Larson
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
@@ -0,0 +1,13 @@
"""Verify certificates using native system trust stores"""
import sys as _sys
if _sys.version_info < (3, 10):
raise ImportError("truststore requires Python 3.10 or later")
from ._api import SSLContext, extract_from_ssl, inject_into_ssl # noqa: E402
del _api, _sys # type: ignore[name-defined] # noqa: F821
__all__ = ["SSLContext", "inject_into_ssl", "extract_from_ssl"]
__version__ = "0.8.0"
@@ -0,0 +1,302 @@
import os
import platform
import socket
import ssl
import typing
import _ssl # type: ignore[import]
from ._ssl_constants import (
_original_SSLContext,
_original_super_SSLContext,
_truststore_SSLContext_dunder_class,
_truststore_SSLContext_super_class,
)
if platform.system() == "Windows":
from ._windows import _configure_context, _verify_peercerts_impl
elif platform.system() == "Darwin":
from ._macos import _configure_context, _verify_peercerts_impl
else:
from ._openssl import _configure_context, _verify_peercerts_impl
if typing.TYPE_CHECKING:
from pipenv.patched.pip._vendor.typing_extensions import Buffer
# From typeshed/stdlib/ssl.pyi
_StrOrBytesPath: typing.TypeAlias = str | bytes | os.PathLike[str] | os.PathLike[bytes]
_PasswordType: typing.TypeAlias = str | bytes | typing.Callable[[], str | bytes]
def inject_into_ssl() -> None:
"""Injects the :class:`truststore.SSLContext` into the ``ssl``
module by replacing :class:`ssl.SSLContext`.
"""
setattr(ssl, "SSLContext", SSLContext)
# urllib3 holds on to its own reference of ssl.SSLContext
# so we need to replace that reference too.
try:
import pipenv.patched.pip._vendor.urllib3.util.ssl_ as urllib3_ssl
setattr(urllib3_ssl, "SSLContext", SSLContext)
except ImportError:
pass
def extract_from_ssl() -> None:
"""Restores the :class:`ssl.SSLContext` class to its original state"""
setattr(ssl, "SSLContext", _original_SSLContext)
try:
import pipenv.patched.pip._vendor.urllib3.util.ssl_ as urllib3_ssl
urllib3_ssl.SSLContext = _original_SSLContext
except ImportError:
pass
class SSLContext(_truststore_SSLContext_super_class): # type: ignore[misc]
"""SSLContext API that uses system certificates on all platforms"""
@property # type: ignore[misc]
def __class__(self) -> type:
# Dirty hack to get around isinstance() checks
# for ssl.SSLContext instances in aiohttp/trustme
# when using non-CPython implementations.
return _truststore_SSLContext_dunder_class or SSLContext
def __init__(self, protocol: int = None) -> None: # type: ignore[assignment]
self._ctx = _original_SSLContext(protocol)
class TruststoreSSLObject(ssl.SSLObject):
# This object exists because wrap_bio() doesn't
# immediately do the handshake so we need to do
# certificate verifications after SSLObject.do_handshake()
def do_handshake(self) -> None:
ret = super().do_handshake()
_verify_peercerts(self, server_hostname=self.server_hostname)
return ret
self._ctx.sslobject_class = TruststoreSSLObject
def wrap_socket(
self,
sock: socket.socket,
server_side: bool = False,
do_handshake_on_connect: bool = True,
suppress_ragged_eofs: bool = True,
server_hostname: str | None = None,
session: ssl.SSLSession | None = None,
) -> ssl.SSLSocket:
# Use a context manager here because the
# inner SSLContext holds on to our state
# but also does the actual handshake.
with _configure_context(self._ctx):
ssl_sock = self._ctx.wrap_socket(
sock,
server_side=server_side,
server_hostname=server_hostname,
do_handshake_on_connect=do_handshake_on_connect,
suppress_ragged_eofs=suppress_ragged_eofs,
session=session,
)
try:
_verify_peercerts(ssl_sock, server_hostname=server_hostname)
except Exception:
ssl_sock.close()
raise
return ssl_sock
def wrap_bio(
self,
incoming: ssl.MemoryBIO,
outgoing: ssl.MemoryBIO,
server_side: bool = False,
server_hostname: str | None = None,
session: ssl.SSLSession | None = None,
) -> ssl.SSLObject:
with _configure_context(self._ctx):
ssl_obj = self._ctx.wrap_bio(
incoming,
outgoing,
server_hostname=server_hostname,
server_side=server_side,
session=session,
)
return ssl_obj
def load_verify_locations(
self,
cafile: str | bytes | os.PathLike[str] | os.PathLike[bytes] | None = None,
capath: str | bytes | os.PathLike[str] | os.PathLike[bytes] | None = None,
cadata: typing.Union[str, "Buffer", None] = None,
) -> None:
return self._ctx.load_verify_locations(
cafile=cafile, capath=capath, cadata=cadata
)
def load_cert_chain(
self,
certfile: _StrOrBytesPath,
keyfile: _StrOrBytesPath | None = None,
password: _PasswordType | None = None,
) -> None:
return self._ctx.load_cert_chain(
certfile=certfile, keyfile=keyfile, password=password
)
def load_default_certs(
self, purpose: ssl.Purpose = ssl.Purpose.SERVER_AUTH
) -> None:
return self._ctx.load_default_certs(purpose)
def set_alpn_protocols(self, alpn_protocols: typing.Iterable[str]) -> None:
return self._ctx.set_alpn_protocols(alpn_protocols)
def set_npn_protocols(self, npn_protocols: typing.Iterable[str]) -> None:
return self._ctx.set_npn_protocols(npn_protocols)
def set_ciphers(self, __cipherlist: str) -> None:
return self._ctx.set_ciphers(__cipherlist)
def get_ciphers(self) -> typing.Any:
return self._ctx.get_ciphers()
def session_stats(self) -> dict[str, int]:
return self._ctx.session_stats()
def cert_store_stats(self) -> dict[str, int]:
raise NotImplementedError()
@typing.overload
def get_ca_certs(
self, binary_form: typing.Literal[False] = ...
) -> list[typing.Any]:
...
@typing.overload
def get_ca_certs(self, binary_form: typing.Literal[True] = ...) -> list[bytes]:
...
@typing.overload
def get_ca_certs(self, binary_form: bool = ...) -> typing.Any:
...
def get_ca_certs(self, binary_form: bool = False) -> list[typing.Any] | list[bytes]:
raise NotImplementedError()
@property
def check_hostname(self) -> bool:
return self._ctx.check_hostname
@check_hostname.setter
def check_hostname(self, value: bool) -> None:
self._ctx.check_hostname = value
@property
def hostname_checks_common_name(self) -> bool:
return self._ctx.hostname_checks_common_name
@hostname_checks_common_name.setter
def hostname_checks_common_name(self, value: bool) -> None:
self._ctx.hostname_checks_common_name = value
@property
def keylog_filename(self) -> str:
return self._ctx.keylog_filename
@keylog_filename.setter
def keylog_filename(self, value: str) -> None:
self._ctx.keylog_filename = value
@property
def maximum_version(self) -> ssl.TLSVersion:
return self._ctx.maximum_version
@maximum_version.setter
def maximum_version(self, value: ssl.TLSVersion) -> None:
_original_super_SSLContext.maximum_version.__set__( # type: ignore[attr-defined]
self._ctx, value
)
@property
def minimum_version(self) -> ssl.TLSVersion:
return self._ctx.minimum_version
@minimum_version.setter
def minimum_version(self, value: ssl.TLSVersion) -> None:
_original_super_SSLContext.minimum_version.__set__( # type: ignore[attr-defined]
self._ctx, value
)
@property
def options(self) -> ssl.Options:
return self._ctx.options
@options.setter
def options(self, value: ssl.Options) -> None:
_original_super_SSLContext.options.__set__( # type: ignore[attr-defined]
self._ctx, value
)
@property
def post_handshake_auth(self) -> bool:
return self._ctx.post_handshake_auth
@post_handshake_auth.setter
def post_handshake_auth(self, value: bool) -> None:
self._ctx.post_handshake_auth = value
@property
def protocol(self) -> ssl._SSLMethod:
return self._ctx.protocol
@property
def security_level(self) -> int:
return self._ctx.security_level
@property
def verify_flags(self) -> ssl.VerifyFlags:
return self._ctx.verify_flags
@verify_flags.setter
def verify_flags(self, value: ssl.VerifyFlags) -> None:
_original_super_SSLContext.verify_flags.__set__( # type: ignore[attr-defined]
self._ctx, value
)
@property
def verify_mode(self) -> ssl.VerifyMode:
return self._ctx.verify_mode
@verify_mode.setter
def verify_mode(self, value: ssl.VerifyMode) -> None:
_original_super_SSLContext.verify_mode.__set__( # type: ignore[attr-defined]
self._ctx, value
)
def _verify_peercerts(
sock_or_sslobj: ssl.SSLSocket | ssl.SSLObject, server_hostname: str | None
) -> None:
"""
Verifies the peer certificates from an SSLSocket or SSLObject
against the certificates in the OS trust store.
"""
sslobj: ssl.SSLObject = sock_or_sslobj # type: ignore[assignment]
try:
while not hasattr(sslobj, "get_unverified_chain"):
sslobj = sslobj._sslobj # type: ignore[attr-defined]
except AttributeError:
pass
# SSLObject.get_unverified_chain() returns 'None'
# if the peer sends no certificates. This is common
# for the server-side scenario.
unverified_chain: typing.Sequence[_ssl.Certificate] = (
sslobj.get_unverified_chain() or () # type: ignore[attr-defined]
)
cert_bytes = [cert.public_bytes(_ssl.ENCODING_DER) for cert in unverified_chain]
_verify_peercerts_impl(
sock_or_sslobj.context, cert_bytes, server_hostname=server_hostname
)
@@ -0,0 +1,501 @@
import contextlib
import ctypes
import platform
import ssl
import typing
from ctypes import (
CDLL,
POINTER,
c_bool,
c_char_p,
c_int32,
c_long,
c_uint32,
c_ulong,
c_void_p,
)
from ctypes.util import find_library
from ._ssl_constants import _set_ssl_context_verify_mode
_mac_version = platform.mac_ver()[0]
_mac_version_info = tuple(map(int, _mac_version.split(".")))
if _mac_version_info < (10, 8):
raise ImportError(
f"Only OS X 10.8 and newer are supported, not {_mac_version_info[0]}.{_mac_version_info[1]}"
)
def _load_cdll(name: str, macos10_16_path: str) -> CDLL:
"""Loads a CDLL by name, falling back to known path on 10.16+"""
try:
# Big Sur is technically 11 but we use 10.16 due to the Big Sur
# beta being labeled as 10.16.
path: str | None
if _mac_version_info >= (10, 16):
path = macos10_16_path
else:
path = find_library(name)
if not path:
raise OSError # Caught and reraised as 'ImportError'
return CDLL(path, use_errno=True)
except OSError:
raise ImportError(f"The library {name} failed to load") from None
Security = _load_cdll(
"Security", "/System/Library/Frameworks/Security.framework/Security"
)
CoreFoundation = _load_cdll(
"CoreFoundation",
"/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation",
)
Boolean = c_bool
CFIndex = c_long
CFStringEncoding = c_uint32
CFData = c_void_p
CFString = c_void_p
CFArray = c_void_p
CFMutableArray = c_void_p
CFError = c_void_p
CFType = c_void_p
CFTypeID = c_ulong
CFTypeRef = POINTER(CFType)
CFAllocatorRef = c_void_p
OSStatus = c_int32
CFErrorRef = POINTER(CFError)
CFDataRef = POINTER(CFData)
CFStringRef = POINTER(CFString)
CFArrayRef = POINTER(CFArray)
CFMutableArrayRef = POINTER(CFMutableArray)
CFArrayCallBacks = c_void_p
CFOptionFlags = c_uint32
SecCertificateRef = POINTER(c_void_p)
SecPolicyRef = POINTER(c_void_p)
SecTrustRef = POINTER(c_void_p)
SecTrustResultType = c_uint32
SecTrustOptionFlags = c_uint32
try:
Security.SecCertificateCreateWithData.argtypes = [CFAllocatorRef, CFDataRef]
Security.SecCertificateCreateWithData.restype = SecCertificateRef
Security.SecCertificateCopyData.argtypes = [SecCertificateRef]
Security.SecCertificateCopyData.restype = CFDataRef
Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p]
Security.SecCopyErrorMessageString.restype = CFStringRef
Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef]
Security.SecTrustSetAnchorCertificates.restype = OSStatus
Security.SecTrustSetAnchorCertificatesOnly.argtypes = [SecTrustRef, Boolean]
Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus
Security.SecTrustEvaluate.argtypes = [SecTrustRef, POINTER(SecTrustResultType)]
Security.SecTrustEvaluate.restype = OSStatus
Security.SecPolicyCreateRevocation.argtypes = [CFOptionFlags]
Security.SecPolicyCreateRevocation.restype = SecPolicyRef
Security.SecPolicyCreateSSL.argtypes = [Boolean, CFStringRef]
Security.SecPolicyCreateSSL.restype = SecPolicyRef
Security.SecTrustCreateWithCertificates.argtypes = [
CFTypeRef,
CFTypeRef,
POINTER(SecTrustRef),
]
Security.SecTrustCreateWithCertificates.restype = OSStatus
Security.SecTrustGetTrustResult.argtypes = [
SecTrustRef,
POINTER(SecTrustResultType),
]
Security.SecTrustGetTrustResult.restype = OSStatus
Security.SecTrustRef = SecTrustRef # type: ignore[attr-defined]
Security.SecTrustResultType = SecTrustResultType # type: ignore[attr-defined]
Security.OSStatus = OSStatus # type: ignore[attr-defined]
kSecRevocationUseAnyAvailableMethod = 3
kSecRevocationRequirePositiveResponse = 8
CoreFoundation.CFRelease.argtypes = [CFTypeRef]
CoreFoundation.CFRelease.restype = None
CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef]
CoreFoundation.CFGetTypeID.restype = CFTypeID
CoreFoundation.CFStringCreateWithCString.argtypes = [
CFAllocatorRef,
c_char_p,
CFStringEncoding,
]
CoreFoundation.CFStringCreateWithCString.restype = CFStringRef
CoreFoundation.CFStringGetCStringPtr.argtypes = [CFStringRef, CFStringEncoding]
CoreFoundation.CFStringGetCStringPtr.restype = c_char_p
CoreFoundation.CFStringGetCString.argtypes = [
CFStringRef,
c_char_p,
CFIndex,
CFStringEncoding,
]
CoreFoundation.CFStringGetCString.restype = c_bool
CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex]
CoreFoundation.CFDataCreate.restype = CFDataRef
CoreFoundation.CFDataGetLength.argtypes = [CFDataRef]
CoreFoundation.CFDataGetLength.restype = CFIndex
CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef]
CoreFoundation.CFDataGetBytePtr.restype = c_void_p
CoreFoundation.CFArrayCreate.argtypes = [
CFAllocatorRef,
POINTER(CFTypeRef),
CFIndex,
CFArrayCallBacks,
]
CoreFoundation.CFArrayCreate.restype = CFArrayRef
CoreFoundation.CFArrayCreateMutable.argtypes = [
CFAllocatorRef,
CFIndex,
CFArrayCallBacks,
]
CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef
CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p]
CoreFoundation.CFArrayAppendValue.restype = None
CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef]
CoreFoundation.CFArrayGetCount.restype = CFIndex
CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex]
CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p
CoreFoundation.CFErrorGetCode.argtypes = [CFErrorRef]
CoreFoundation.CFErrorGetCode.restype = CFIndex
CoreFoundation.CFErrorCopyDescription.argtypes = [CFErrorRef]
CoreFoundation.CFErrorCopyDescription.restype = CFStringRef
CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll( # type: ignore[attr-defined]
CoreFoundation, "kCFAllocatorDefault"
)
CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll( # type: ignore[attr-defined]
CoreFoundation, "kCFTypeArrayCallBacks"
)
CoreFoundation.CFTypeRef = CFTypeRef # type: ignore[attr-defined]
CoreFoundation.CFArrayRef = CFArrayRef # type: ignore[attr-defined]
CoreFoundation.CFStringRef = CFStringRef # type: ignore[attr-defined]
CoreFoundation.CFErrorRef = CFErrorRef # type: ignore[attr-defined]
except AttributeError:
raise ImportError("Error initializing ctypes") from None
def _handle_osstatus(result: OSStatus, _: typing.Any, args: typing.Any) -> typing.Any:
"""
Raises an error if the OSStatus value is non-zero.
"""
if int(result) == 0:
return args
# Returns a CFString which we need to transform
# into a UTF-8 Python string.
error_message_cfstring = None
try:
error_message_cfstring = Security.SecCopyErrorMessageString(result, None)
# First step is convert the CFString into a C string pointer.
# We try the fast no-copy way first.
error_message_cfstring_c_void_p = ctypes.cast(
error_message_cfstring, ctypes.POINTER(ctypes.c_void_p)
)
message = CoreFoundation.CFStringGetCStringPtr(
error_message_cfstring_c_void_p, CFConst.kCFStringEncodingUTF8
)
# Quoting the Apple dev docs:
#
# "A pointer to a C string or NULL if the internal
# storage of theString does not allow this to be
# returned efficiently."
#
# So we need to get our hands dirty.
if message is None:
buffer = ctypes.create_string_buffer(1024)
result = CoreFoundation.CFStringGetCString(
error_message_cfstring_c_void_p,
buffer,
1024,
CFConst.kCFStringEncodingUTF8,
)
if not result:
raise OSError("Error copying C string from CFStringRef")
message = buffer.value
finally:
if error_message_cfstring is not None:
CoreFoundation.CFRelease(error_message_cfstring)
# If no message can be found for this status we come
# up with a generic one that forwards the status code.
if message is None or message == "":
message = f"SecureTransport operation returned a non-zero OSStatus: {result}"
raise ssl.SSLError(message)
Security.SecTrustCreateWithCertificates.errcheck = _handle_osstatus # type: ignore[assignment]
Security.SecTrustSetAnchorCertificates.errcheck = _handle_osstatus # type: ignore[assignment]
Security.SecTrustGetTrustResult.errcheck = _handle_osstatus # type: ignore[assignment]
class CFConst:
"""CoreFoundation constants"""
kCFStringEncodingUTF8 = CFStringEncoding(0x08000100)
errSecIncompleteCertRevocationCheck = -67635
errSecHostNameMismatch = -67602
errSecCertificateExpired = -67818
errSecNotTrusted = -67843
def _bytes_to_cf_data_ref(value: bytes) -> CFDataRef: # type: ignore[valid-type]
return CoreFoundation.CFDataCreate( # type: ignore[no-any-return]
CoreFoundation.kCFAllocatorDefault, value, len(value)
)
def _bytes_to_cf_string(value: bytes) -> CFString:
"""
Given a Python binary data, create a CFString.
The string must be CFReleased by the caller.
"""
c_str = ctypes.c_char_p(value)
cf_str = CoreFoundation.CFStringCreateWithCString(
CoreFoundation.kCFAllocatorDefault,
c_str,
CFConst.kCFStringEncodingUTF8,
)
return cf_str # type: ignore[no-any-return]
def _cf_string_ref_to_str(cf_string_ref: CFStringRef) -> str | None: # type: ignore[valid-type]
"""
Creates a Unicode string from a CFString object. Used entirely for error
reporting.
Yes, it annoys me quite a lot that this function is this complex.
"""
string = CoreFoundation.CFStringGetCStringPtr(
cf_string_ref, CFConst.kCFStringEncodingUTF8
)
if string is None:
buffer = ctypes.create_string_buffer(1024)
result = CoreFoundation.CFStringGetCString(
cf_string_ref, buffer, 1024, CFConst.kCFStringEncodingUTF8
)
if not result:
raise OSError("Error copying C string from CFStringRef")
string = buffer.value
if string is not None:
string = string.decode("utf-8")
return string # type: ignore[no-any-return]
def _der_certs_to_cf_cert_array(certs: list[bytes]) -> CFMutableArrayRef: # type: ignore[valid-type]
"""Builds a CFArray of SecCertificateRefs from a list of DER-encoded certificates.
Responsibility of the caller to call CoreFoundation.CFRelease on the CFArray.
"""
cf_array = CoreFoundation.CFArrayCreateMutable(
CoreFoundation.kCFAllocatorDefault,
0,
ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks),
)
if not cf_array:
raise MemoryError("Unable to allocate memory!")
for cert_data in certs:
cf_data = None
sec_cert_ref = None
try:
cf_data = _bytes_to_cf_data_ref(cert_data)
sec_cert_ref = Security.SecCertificateCreateWithData(
CoreFoundation.kCFAllocatorDefault, cf_data
)
CoreFoundation.CFArrayAppendValue(cf_array, sec_cert_ref)
finally:
if cf_data:
CoreFoundation.CFRelease(cf_data)
if sec_cert_ref:
CoreFoundation.CFRelease(sec_cert_ref)
return cf_array # type: ignore[no-any-return]
@contextlib.contextmanager
def _configure_context(ctx: ssl.SSLContext) -> typing.Iterator[None]:
check_hostname = ctx.check_hostname
verify_mode = ctx.verify_mode
ctx.check_hostname = False
_set_ssl_context_verify_mode(ctx, ssl.CERT_NONE)
try:
yield
finally:
ctx.check_hostname = check_hostname
_set_ssl_context_verify_mode(ctx, verify_mode)
def _verify_peercerts_impl(
ssl_context: ssl.SSLContext,
cert_chain: list[bytes],
server_hostname: str | None = None,
) -> None:
certs = None
policies = None
trust = None
cf_error = None
try:
if server_hostname is not None:
cf_str_hostname = None
try:
cf_str_hostname = _bytes_to_cf_string(server_hostname.encode("ascii"))
ssl_policy = Security.SecPolicyCreateSSL(True, cf_str_hostname)
finally:
if cf_str_hostname:
CoreFoundation.CFRelease(cf_str_hostname)
else:
ssl_policy = Security.SecPolicyCreateSSL(True, None)
policies = ssl_policy
if ssl_context.verify_flags & ssl.VERIFY_CRL_CHECK_CHAIN:
# Add explicit policy requiring positive revocation checks
policies = CoreFoundation.CFArrayCreateMutable(
CoreFoundation.kCFAllocatorDefault,
0,
ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks),
)
CoreFoundation.CFArrayAppendValue(policies, ssl_policy)
CoreFoundation.CFRelease(ssl_policy)
revocation_policy = Security.SecPolicyCreateRevocation(
kSecRevocationUseAnyAvailableMethod
| kSecRevocationRequirePositiveResponse
)
CoreFoundation.CFArrayAppendValue(policies, revocation_policy)
CoreFoundation.CFRelease(revocation_policy)
elif ssl_context.verify_flags & ssl.VERIFY_CRL_CHECK_LEAF:
raise NotImplementedError("VERIFY_CRL_CHECK_LEAF not implemented for macOS")
certs = None
try:
certs = _der_certs_to_cf_cert_array(cert_chain)
# Now that we have certificates loaded and a SecPolicy
# we can finally create a SecTrust object!
trust = Security.SecTrustRef()
Security.SecTrustCreateWithCertificates(
certs, policies, ctypes.byref(trust)
)
finally:
# The certs are now being held by SecTrust so we can
# release our handles for the array.
if certs:
CoreFoundation.CFRelease(certs)
# If there are additional trust anchors to load we need to transform
# the list of DER-encoded certificates into a CFArray. Otherwise
# pass 'None' to signal that we only want system / fetched certificates.
ctx_ca_certs_der: list[bytes] | None = ssl_context.get_ca_certs(
binary_form=True
)
if ctx_ca_certs_der:
ctx_ca_certs = None
try:
ctx_ca_certs = _der_certs_to_cf_cert_array(cert_chain)
Security.SecTrustSetAnchorCertificates(trust, ctx_ca_certs)
finally:
if ctx_ca_certs:
CoreFoundation.CFRelease(ctx_ca_certs)
else:
Security.SecTrustSetAnchorCertificates(trust, None)
cf_error = CoreFoundation.CFErrorRef()
sec_trust_eval_result = Security.SecTrustEvaluateWithError(
trust, ctypes.byref(cf_error)
)
# sec_trust_eval_result is a bool (0 or 1)
# where 1 means that the certs are trusted.
if sec_trust_eval_result == 1:
is_trusted = True
elif sec_trust_eval_result == 0:
is_trusted = False
else:
raise ssl.SSLError(
f"Unknown result from Security.SecTrustEvaluateWithError: {sec_trust_eval_result!r}"
)
cf_error_code = 0
if not is_trusted:
cf_error_code = CoreFoundation.CFErrorGetCode(cf_error)
# If the error is a known failure that we're
# explicitly okay with from SSLContext configuration
# we can set is_trusted accordingly.
if ssl_context.verify_mode != ssl.CERT_REQUIRED and (
cf_error_code == CFConst.errSecNotTrusted
or cf_error_code == CFConst.errSecCertificateExpired
):
is_trusted = True
elif (
not ssl_context.check_hostname
and cf_error_code == CFConst.errSecHostNameMismatch
):
is_trusted = True
# If we're still not trusted then we start to
# construct and raise the SSLCertVerificationError.
if not is_trusted:
cf_error_string_ref = None
try:
cf_error_string_ref = CoreFoundation.CFErrorCopyDescription(cf_error)
# Can this ever return 'None' if there's a CFError?
cf_error_message = (
_cf_string_ref_to_str(cf_error_string_ref)
or "Certificate verification failed"
)
# TODO: Not sure if we need the SecTrustResultType for anything?
# We only care whether or not it's a success or failure for now.
sec_trust_result_type = Security.SecTrustResultType()
Security.SecTrustGetTrustResult(
trust, ctypes.byref(sec_trust_result_type)
)
err = ssl.SSLCertVerificationError(cf_error_message)
err.verify_message = cf_error_message
err.verify_code = cf_error_code
raise err
finally:
if cf_error_string_ref:
CoreFoundation.CFRelease(cf_error_string_ref)
finally:
if policies:
CoreFoundation.CFRelease(policies)
if trust:
CoreFoundation.CFRelease(trust)
@@ -0,0 +1,66 @@
import contextlib
import os
import re
import ssl
import typing
# candidates based on https://github.com/tiran/certifi-system-store by Christian Heimes
_CA_FILE_CANDIDATES = [
# Alpine, Arch, Fedora 34+, OpenWRT, RHEL 9+, BSD
"/etc/ssl/cert.pem",
# Fedora <= 34, RHEL <= 9, CentOS <= 9
"/etc/pki/tls/cert.pem",
# Debian, Ubuntu (requires ca-certificates)
"/etc/ssl/certs/ca-certificates.crt",
# SUSE
"/etc/ssl/ca-bundle.pem",
]
_HASHED_CERT_FILENAME_RE = re.compile(r"^[0-9a-fA-F]{8}\.[0-9]$")
@contextlib.contextmanager
def _configure_context(ctx: ssl.SSLContext) -> typing.Iterator[None]:
# First, check whether the default locations from OpenSSL
# seem like they will give us a usable set of CA certs.
# ssl.get_default_verify_paths already takes care of:
# - getting cafile from either the SSL_CERT_FILE env var
# or the path configured when OpenSSL was compiled,
# and verifying that that path exists
# - getting capath from either the SSL_CERT_DIR env var
# or the path configured when OpenSSL was compiled,
# and verifying that that path exists
# In addition we'll check whether capath appears to contain certs.
defaults = ssl.get_default_verify_paths()
if defaults.cafile or (defaults.capath and _capath_contains_certs(defaults.capath)):
ctx.set_default_verify_paths()
else:
# cafile from OpenSSL doesn't exist
# and capath from OpenSSL doesn't contain certs.
# Let's search other common locations instead.
for cafile in _CA_FILE_CANDIDATES:
if os.path.isfile(cafile):
ctx.load_verify_locations(cafile=cafile)
break
yield
def _capath_contains_certs(capath: str) -> bool:
"""Check whether capath exists and contains certs in the expected format."""
if not os.path.isdir(capath):
return False
for name in os.listdir(capath):
if _HASHED_CERT_FILENAME_RE.match(name):
return True
return False
def _verify_peercerts_impl(
ssl_context: ssl.SSLContext,
cert_chain: list[bytes],
server_hostname: str | None = None,
) -> None:
# This is a no-op because we've enabled SSLContext's built-in
# verification via verify_mode=CERT_REQUIRED, and don't need to repeat it.
pass
@@ -0,0 +1,31 @@
import ssl
import sys
import typing
# Hold on to the original class so we can create it consistently
# even if we inject our own SSLContext into the ssl module.
_original_SSLContext = ssl.SSLContext
_original_super_SSLContext = super(_original_SSLContext, _original_SSLContext)
# CPython is known to be good, but non-CPython implementations
# may implement SSLContext differently so to be safe we don't
# subclass the SSLContext.
# This is returned by truststore.SSLContext.__class__()
_truststore_SSLContext_dunder_class: typing.Optional[type]
# This value is the superclass of truststore.SSLContext.
_truststore_SSLContext_super_class: type
if sys.implementation.name == "cpython":
_truststore_SSLContext_super_class = _original_SSLContext
_truststore_SSLContext_dunder_class = None
else:
_truststore_SSLContext_super_class = object
_truststore_SSLContext_dunder_class = _original_SSLContext
def _set_ssl_context_verify_mode(
ssl_context: ssl.SSLContext, verify_mode: ssl.VerifyMode
) -> None:
_original_super_SSLContext.verify_mode.__set__(ssl_context, verify_mode) # type: ignore[attr-defined]
@@ -0,0 +1,554 @@
import contextlib
import ssl
import typing
from ctypes import WinDLL # type: ignore
from ctypes import WinError # type: ignore
from ctypes import (
POINTER,
Structure,
c_char_p,
c_ulong,
c_void_p,
c_wchar_p,
cast,
create_unicode_buffer,
pointer,
sizeof,
)
from ctypes.wintypes import (
BOOL,
DWORD,
HANDLE,
LONG,
LPCSTR,
LPCVOID,
LPCWSTR,
LPFILETIME,
LPSTR,
LPWSTR,
)
from typing import TYPE_CHECKING, Any
from ._ssl_constants import _set_ssl_context_verify_mode
HCERTCHAINENGINE = HANDLE
HCERTSTORE = HANDLE
HCRYPTPROV_LEGACY = HANDLE
class CERT_CONTEXT(Structure):
_fields_ = (
("dwCertEncodingType", DWORD),
("pbCertEncoded", c_void_p),
("cbCertEncoded", DWORD),
("pCertInfo", c_void_p),
("hCertStore", HCERTSTORE),
)
PCERT_CONTEXT = POINTER(CERT_CONTEXT)
PCCERT_CONTEXT = POINTER(PCERT_CONTEXT)
class CERT_ENHKEY_USAGE(Structure):
_fields_ = (
("cUsageIdentifier", DWORD),
("rgpszUsageIdentifier", POINTER(LPSTR)),
)
PCERT_ENHKEY_USAGE = POINTER(CERT_ENHKEY_USAGE)
class CERT_USAGE_MATCH(Structure):
_fields_ = (
("dwType", DWORD),
("Usage", CERT_ENHKEY_USAGE),
)
class CERT_CHAIN_PARA(Structure):
_fields_ = (
("cbSize", DWORD),
("RequestedUsage", CERT_USAGE_MATCH),
("RequestedIssuancePolicy", CERT_USAGE_MATCH),
("dwUrlRetrievalTimeout", DWORD),
("fCheckRevocationFreshnessTime", BOOL),
("dwRevocationFreshnessTime", DWORD),
("pftCacheResync", LPFILETIME),
("pStrongSignPara", c_void_p),
("dwStrongSignFlags", DWORD),
)
if TYPE_CHECKING:
PCERT_CHAIN_PARA = pointer[CERT_CHAIN_PARA] # type: ignore[misc]
else:
PCERT_CHAIN_PARA = POINTER(CERT_CHAIN_PARA)
class CERT_TRUST_STATUS(Structure):
_fields_ = (
("dwErrorStatus", DWORD),
("dwInfoStatus", DWORD),
)
class CERT_CHAIN_ELEMENT(Structure):
_fields_ = (
("cbSize", DWORD),
("pCertContext", PCERT_CONTEXT),
("TrustStatus", CERT_TRUST_STATUS),
("pRevocationInfo", c_void_p),
("pIssuanceUsage", PCERT_ENHKEY_USAGE),
("pApplicationUsage", PCERT_ENHKEY_USAGE),
("pwszExtendedErrorInfo", LPCWSTR),
)
PCERT_CHAIN_ELEMENT = POINTER(CERT_CHAIN_ELEMENT)
class CERT_SIMPLE_CHAIN(Structure):
_fields_ = (
("cbSize", DWORD),
("TrustStatus", CERT_TRUST_STATUS),
("cElement", DWORD),
("rgpElement", POINTER(PCERT_CHAIN_ELEMENT)),
("pTrustListInfo", c_void_p),
("fHasRevocationFreshnessTime", BOOL),
("dwRevocationFreshnessTime", DWORD),
)
PCERT_SIMPLE_CHAIN = POINTER(CERT_SIMPLE_CHAIN)
class CERT_CHAIN_CONTEXT(Structure):
_fields_ = (
("cbSize", DWORD),
("TrustStatus", CERT_TRUST_STATUS),
("cChain", DWORD),
("rgpChain", POINTER(PCERT_SIMPLE_CHAIN)),
("cLowerQualityChainContext", DWORD),
("rgpLowerQualityChainContext", c_void_p),
("fHasRevocationFreshnessTime", BOOL),
("dwRevocationFreshnessTime", DWORD),
)
PCERT_CHAIN_CONTEXT = POINTER(CERT_CHAIN_CONTEXT)
PCCERT_CHAIN_CONTEXT = POINTER(PCERT_CHAIN_CONTEXT)
class SSL_EXTRA_CERT_CHAIN_POLICY_PARA(Structure):
_fields_ = (
("cbSize", DWORD),
("dwAuthType", DWORD),
("fdwChecks", DWORD),
("pwszServerName", LPCWSTR),
)
class CERT_CHAIN_POLICY_PARA(Structure):
_fields_ = (
("cbSize", DWORD),
("dwFlags", DWORD),
("pvExtraPolicyPara", c_void_p),
)
PCERT_CHAIN_POLICY_PARA = POINTER(CERT_CHAIN_POLICY_PARA)
class CERT_CHAIN_POLICY_STATUS(Structure):
_fields_ = (
("cbSize", DWORD),
("dwError", DWORD),
("lChainIndex", LONG),
("lElementIndex", LONG),
("pvExtraPolicyStatus", c_void_p),
)
PCERT_CHAIN_POLICY_STATUS = POINTER(CERT_CHAIN_POLICY_STATUS)
class CERT_CHAIN_ENGINE_CONFIG(Structure):
_fields_ = (
("cbSize", DWORD),
("hRestrictedRoot", HCERTSTORE),
("hRestrictedTrust", HCERTSTORE),
("hRestrictedOther", HCERTSTORE),
("cAdditionalStore", DWORD),
("rghAdditionalStore", c_void_p),
("dwFlags", DWORD),
("dwUrlRetrievalTimeout", DWORD),
("MaximumCachedCertificates", DWORD),
("CycleDetectionModulus", DWORD),
("hExclusiveRoot", HCERTSTORE),
("hExclusiveTrustedPeople", HCERTSTORE),
("dwExclusiveFlags", DWORD),
)
PCERT_CHAIN_ENGINE_CONFIG = POINTER(CERT_CHAIN_ENGINE_CONFIG)
PHCERTCHAINENGINE = POINTER(HCERTCHAINENGINE)
X509_ASN_ENCODING = 0x00000001
PKCS_7_ASN_ENCODING = 0x00010000
CERT_STORE_PROV_MEMORY = b"Memory"
CERT_STORE_ADD_USE_EXISTING = 2
USAGE_MATCH_TYPE_OR = 1
OID_PKIX_KP_SERVER_AUTH = c_char_p(b"1.3.6.1.5.5.7.3.1")
CERT_CHAIN_REVOCATION_CHECK_END_CERT = 0x10000000
CERT_CHAIN_REVOCATION_CHECK_CHAIN = 0x20000000
CERT_CHAIN_POLICY_IGNORE_ALL_NOT_TIME_VALID_FLAGS = 0x00000007
CERT_CHAIN_POLICY_IGNORE_INVALID_BASIC_CONSTRAINTS_FLAG = 0x00000008
CERT_CHAIN_POLICY_ALLOW_UNKNOWN_CA_FLAG = 0x00000010
CERT_CHAIN_POLICY_IGNORE_INVALID_NAME_FLAG = 0x00000040
CERT_CHAIN_POLICY_IGNORE_WRONG_USAGE_FLAG = 0x00000020
CERT_CHAIN_POLICY_IGNORE_INVALID_POLICY_FLAG = 0x00000080
CERT_CHAIN_POLICY_IGNORE_ALL_REV_UNKNOWN_FLAGS = 0x00000F00
CERT_CHAIN_POLICY_ALLOW_TESTROOT_FLAG = 0x00008000
CERT_CHAIN_POLICY_TRUST_TESTROOT_FLAG = 0x00004000
AUTHTYPE_SERVER = 2
CERT_CHAIN_POLICY_SSL = 4
FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000
FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200
# Flags to set for SSLContext.verify_mode=CERT_NONE
CERT_CHAIN_POLICY_VERIFY_MODE_NONE_FLAGS = (
CERT_CHAIN_POLICY_IGNORE_ALL_NOT_TIME_VALID_FLAGS
| CERT_CHAIN_POLICY_IGNORE_INVALID_BASIC_CONSTRAINTS_FLAG
| CERT_CHAIN_POLICY_ALLOW_UNKNOWN_CA_FLAG
| CERT_CHAIN_POLICY_IGNORE_INVALID_NAME_FLAG
| CERT_CHAIN_POLICY_IGNORE_WRONG_USAGE_FLAG
| CERT_CHAIN_POLICY_IGNORE_INVALID_POLICY_FLAG
| CERT_CHAIN_POLICY_IGNORE_ALL_REV_UNKNOWN_FLAGS
| CERT_CHAIN_POLICY_ALLOW_TESTROOT_FLAG
| CERT_CHAIN_POLICY_TRUST_TESTROOT_FLAG
)
wincrypt = WinDLL("crypt32.dll")
kernel32 = WinDLL("kernel32.dll")
def _handle_win_error(result: bool, _: Any, args: Any) -> Any:
if not result:
# Note, actually raises OSError after calling GetLastError and FormatMessage
raise WinError()
return args
CertCreateCertificateChainEngine = wincrypt.CertCreateCertificateChainEngine
CertCreateCertificateChainEngine.argtypes = (
PCERT_CHAIN_ENGINE_CONFIG,
PHCERTCHAINENGINE,
)
CertCreateCertificateChainEngine.errcheck = _handle_win_error
CertOpenStore = wincrypt.CertOpenStore
CertOpenStore.argtypes = (LPCSTR, DWORD, HCRYPTPROV_LEGACY, DWORD, c_void_p)
CertOpenStore.restype = HCERTSTORE
CertOpenStore.errcheck = _handle_win_error
CertAddEncodedCertificateToStore = wincrypt.CertAddEncodedCertificateToStore
CertAddEncodedCertificateToStore.argtypes = (
HCERTSTORE,
DWORD,
c_char_p,
DWORD,
DWORD,
PCCERT_CONTEXT,
)
CertAddEncodedCertificateToStore.restype = BOOL
CertCreateCertificateContext = wincrypt.CertCreateCertificateContext
CertCreateCertificateContext.argtypes = (DWORD, c_char_p, DWORD)
CertCreateCertificateContext.restype = PCERT_CONTEXT
CertCreateCertificateContext.errcheck = _handle_win_error
CertGetCertificateChain = wincrypt.CertGetCertificateChain
CertGetCertificateChain.argtypes = (
HCERTCHAINENGINE,
PCERT_CONTEXT,
LPFILETIME,
HCERTSTORE,
PCERT_CHAIN_PARA,
DWORD,
c_void_p,
PCCERT_CHAIN_CONTEXT,
)
CertGetCertificateChain.restype = BOOL
CertGetCertificateChain.errcheck = _handle_win_error
CertVerifyCertificateChainPolicy = wincrypt.CertVerifyCertificateChainPolicy
CertVerifyCertificateChainPolicy.argtypes = (
c_ulong,
PCERT_CHAIN_CONTEXT,
PCERT_CHAIN_POLICY_PARA,
PCERT_CHAIN_POLICY_STATUS,
)
CertVerifyCertificateChainPolicy.restype = BOOL
CertCloseStore = wincrypt.CertCloseStore
CertCloseStore.argtypes = (HCERTSTORE, DWORD)
CertCloseStore.restype = BOOL
CertCloseStore.errcheck = _handle_win_error
CertFreeCertificateChain = wincrypt.CertFreeCertificateChain
CertFreeCertificateChain.argtypes = (PCERT_CHAIN_CONTEXT,)
CertFreeCertificateContext = wincrypt.CertFreeCertificateContext
CertFreeCertificateContext.argtypes = (PCERT_CONTEXT,)
CertFreeCertificateChainEngine = wincrypt.CertFreeCertificateChainEngine
CertFreeCertificateChainEngine.argtypes = (HCERTCHAINENGINE,)
FormatMessageW = kernel32.FormatMessageW
FormatMessageW.argtypes = (
DWORD,
LPCVOID,
DWORD,
DWORD,
LPWSTR,
DWORD,
c_void_p,
)
FormatMessageW.restype = DWORD
def _verify_peercerts_impl(
ssl_context: ssl.SSLContext,
cert_chain: list[bytes],
server_hostname: str | None = None,
) -> None:
"""Verify the cert_chain from the server using Windows APIs."""
pCertContext = None
hIntermediateCertStore = CertOpenStore(CERT_STORE_PROV_MEMORY, 0, None, 0, None)
try:
# Add intermediate certs to an in-memory cert store
for cert_bytes in cert_chain[1:]:
CertAddEncodedCertificateToStore(
hIntermediateCertStore,
X509_ASN_ENCODING | PKCS_7_ASN_ENCODING,
cert_bytes,
len(cert_bytes),
CERT_STORE_ADD_USE_EXISTING,
None,
)
# Cert context for leaf cert
leaf_cert = cert_chain[0]
pCertContext = CertCreateCertificateContext(
X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, leaf_cert, len(leaf_cert)
)
# Chain params to match certs for serverAuth extended usage
cert_enhkey_usage = CERT_ENHKEY_USAGE()
cert_enhkey_usage.cUsageIdentifier = 1
cert_enhkey_usage.rgpszUsageIdentifier = (c_char_p * 1)(OID_PKIX_KP_SERVER_AUTH)
cert_usage_match = CERT_USAGE_MATCH()
cert_usage_match.Usage = cert_enhkey_usage
chain_params = CERT_CHAIN_PARA()
chain_params.RequestedUsage = cert_usage_match
chain_params.cbSize = sizeof(chain_params)
pChainPara = pointer(chain_params)
if ssl_context.verify_flags & ssl.VERIFY_CRL_CHECK_CHAIN:
chain_flags = CERT_CHAIN_REVOCATION_CHECK_CHAIN
elif ssl_context.verify_flags & ssl.VERIFY_CRL_CHECK_LEAF:
chain_flags = CERT_CHAIN_REVOCATION_CHECK_END_CERT
else:
chain_flags = 0
try:
# First attempt to verify using the default Windows system trust roots
# (default chain engine).
_get_and_verify_cert_chain(
ssl_context,
None,
hIntermediateCertStore,
pCertContext,
pChainPara,
server_hostname,
chain_flags=chain_flags,
)
except ssl.SSLCertVerificationError:
# If that fails but custom CA certs have been added
# to the SSLContext using load_verify_locations,
# try verifying using a custom chain engine
# that trusts the custom CA certs.
custom_ca_certs: list[bytes] | None = ssl_context.get_ca_certs(
binary_form=True
)
if custom_ca_certs:
_verify_using_custom_ca_certs(
ssl_context,
custom_ca_certs,
hIntermediateCertStore,
pCertContext,
pChainPara,
server_hostname,
chain_flags=chain_flags,
)
else:
raise
finally:
CertCloseStore(hIntermediateCertStore, 0)
if pCertContext:
CertFreeCertificateContext(pCertContext)
def _get_and_verify_cert_chain(
ssl_context: ssl.SSLContext,
hChainEngine: HCERTCHAINENGINE | None,
hIntermediateCertStore: HCERTSTORE,
pPeerCertContext: c_void_p,
pChainPara: PCERT_CHAIN_PARA, # type: ignore[valid-type]
server_hostname: str | None,
chain_flags: int,
) -> None:
ppChainContext = None
try:
# Get cert chain
ppChainContext = pointer(PCERT_CHAIN_CONTEXT())
CertGetCertificateChain(
hChainEngine, # chain engine
pPeerCertContext, # leaf cert context
None, # current system time
hIntermediateCertStore, # additional in-memory cert store
pChainPara, # chain-building parameters
chain_flags,
None, # reserved
ppChainContext, # the resulting chain context
)
pChainContext = ppChainContext.contents
# Verify cert chain
ssl_extra_cert_chain_policy_para = SSL_EXTRA_CERT_CHAIN_POLICY_PARA()
ssl_extra_cert_chain_policy_para.cbSize = sizeof(
ssl_extra_cert_chain_policy_para
)
ssl_extra_cert_chain_policy_para.dwAuthType = AUTHTYPE_SERVER
ssl_extra_cert_chain_policy_para.fdwChecks = 0
if server_hostname:
ssl_extra_cert_chain_policy_para.pwszServerName = c_wchar_p(server_hostname)
chain_policy = CERT_CHAIN_POLICY_PARA()
chain_policy.pvExtraPolicyPara = cast(
pointer(ssl_extra_cert_chain_policy_para), c_void_p
)
if ssl_context.verify_mode == ssl.CERT_NONE:
chain_policy.dwFlags |= CERT_CHAIN_POLICY_VERIFY_MODE_NONE_FLAGS
if not ssl_context.check_hostname:
chain_policy.dwFlags |= CERT_CHAIN_POLICY_IGNORE_INVALID_NAME_FLAG
chain_policy.cbSize = sizeof(chain_policy)
pPolicyPara = pointer(chain_policy)
policy_status = CERT_CHAIN_POLICY_STATUS()
policy_status.cbSize = sizeof(policy_status)
pPolicyStatus = pointer(policy_status)
CertVerifyCertificateChainPolicy(
CERT_CHAIN_POLICY_SSL,
pChainContext,
pPolicyPara,
pPolicyStatus,
)
# Check status
error_code = policy_status.dwError
if error_code:
# Try getting a human readable message for an error code.
error_message_buf = create_unicode_buffer(1024)
error_message_chars = FormatMessageW(
FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
None,
error_code,
0,
error_message_buf,
sizeof(error_message_buf),
None,
)
# See if we received a message for the error,
# otherwise we use a generic error with the
# error code and hope that it's search-able.
if error_message_chars <= 0:
error_message = f"Certificate chain policy error {error_code:#x} [{policy_status.lElementIndex}]"
else:
error_message = error_message_buf.value.strip()
err = ssl.SSLCertVerificationError(error_message)
err.verify_message = error_message
err.verify_code = error_code
raise err from None
finally:
if ppChainContext:
CertFreeCertificateChain(ppChainContext.contents)
def _verify_using_custom_ca_certs(
ssl_context: ssl.SSLContext,
custom_ca_certs: list[bytes],
hIntermediateCertStore: HCERTSTORE,
pPeerCertContext: c_void_p,
pChainPara: PCERT_CHAIN_PARA, # type: ignore[valid-type]
server_hostname: str | None,
chain_flags: int,
) -> None:
hChainEngine = None
hRootCertStore = CertOpenStore(CERT_STORE_PROV_MEMORY, 0, None, 0, None)
try:
# Add custom CA certs to an in-memory cert store
for cert_bytes in custom_ca_certs:
CertAddEncodedCertificateToStore(
hRootCertStore,
X509_ASN_ENCODING | PKCS_7_ASN_ENCODING,
cert_bytes,
len(cert_bytes),
CERT_STORE_ADD_USE_EXISTING,
None,
)
# Create a custom cert chain engine which exclusively trusts
# certs from our hRootCertStore
cert_chain_engine_config = CERT_CHAIN_ENGINE_CONFIG()
cert_chain_engine_config.cbSize = sizeof(cert_chain_engine_config)
cert_chain_engine_config.hExclusiveRoot = hRootCertStore
pConfig = pointer(cert_chain_engine_config)
phChainEngine = pointer(HCERTCHAINENGINE())
CertCreateCertificateChainEngine(
pConfig,
phChainEngine,
)
hChainEngine = phChainEngine.contents
# Get and verify a cert chain using the custom chain engine
_get_and_verify_cert_chain(
ssl_context,
hChainEngine,
hIntermediateCertStore,
pPeerCertContext,
pChainPara,
server_hostname,
chain_flags,
)
finally:
if hChainEngine:
CertFreeCertificateChainEngine(hChainEngine)
CertCloseStore(hRootCertStore, 0)
@contextlib.contextmanager
def _configure_context(ctx: ssl.SSLContext) -> typing.Iterator[None]:
check_hostname = ctx.check_hostname
verify_mode = ctx.verify_mode
ctx.check_hostname = False
_set_ssl_context_verify_mode(ctx, ssl.CERT_NONE)
try:
yield
finally:
ctx.check_hostname = check_hostname
_set_ssl_context_verify_mode(ctx, verify_mode)
@@ -1,2 +1,2 @@
# This file is protected via CODEOWNERS
__version__ = "1.26.16"
__version__ = "1.26.17"
@@ -1,6 +1,9 @@
from __future__ import absolute_import
import sys
from .filepost import encode_multipart_formdata
from .packages import six
from .packages.six.moves.urllib.parse import urlencode
__all__ = ["RequestMethods"]
@@ -168,3 +171,21 @@ class RequestMethods(object):
extra_kw.update(urlopen_kw)
return self.urlopen(method, url, **extra_kw)
if not six.PY2:
class RequestModule(sys.modules[__name__].__class__):
def __call__(self, *args, **kwargs):
"""
If user tries to call this module directly urllib3 v2.x style raise an error to the user
suggesting they may need urllib3 v2
"""
raise TypeError(
"'module' object is not callable\n"
"urllib3.request() method is not supported in this release, "
"upgrade to urllib3 v2 to use it\n"
"see https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html"
)
sys.modules[__name__].__class__ = RequestModule
@@ -235,7 +235,7 @@ class Retry(object):
RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
#: Default headers to be used for ``remove_headers_on_redirect``
DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"])
DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"])
#: Maximum backoff time.
DEFAULT_BACKOFF_MAX = 120
+4 -3
View File
@@ -1,4 +1,4 @@
CacheControl==0.12.11 # Make sure to update the license in pyproject.toml for this.
CacheControl==0.13.1 # Make sure to update the license in pyproject.toml for this.
colorama==0.4.6
distlib==0.3.6
distro==1.8.0
@@ -8,10 +8,10 @@ platformdirs==3.8.1
pyparsing==3.1.0
pyproject-hooks==1.0.0
requests==2.31.0
certifi==2023.5.7
certifi==2023.7.22
chardet==5.1.0
idna==3.4
urllib3==1.26.16
urllib3==1.26.17
rich==13.4.2
pygments==2.15.1
typing_extensions==4.7.1
@@ -20,4 +20,5 @@ setuptools==68.0.0
six==1.16.0
tenacity==8.2.2
tomli==2.0.1
truststore==0.8.0
webencodings==0.5.1