From 729e790ef4dfa91ac9db789b7621039d2a4c264a Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 19 Mar 2019 01:12:54 -0400 Subject: [PATCH 01/22] Update pip version Signed-off-by: Dan Ryan --- pipenv/patched/patched.txt | 2 +- .../patched/_post-pip-update-pep425tags.patch | 13 +- .../patched/_post-pip-update-pypi-uri.patch | 4 +- .../_post-pip-update-requests-imports.patch | 2 +- .../patched/{pip18.patch => pip19.patch} | 838 +++++++++--------- 5 files changed, 423 insertions(+), 436 deletions(-) rename tasks/vendoring/patches/patched/{pip18.patch => pip19.patch} (74%) diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index e7dadd8e..41cb516e 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -2,4 +2,4 @@ safety crayons==0.1.2 pipfile==0.0.2 pip-tools==3.1.0 -pip==18.1 +pip==19.0.3 diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch index fce6ae89..b4ffbc9f 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch @@ -1,8 +1,8 @@ diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py -index f3b9b5b4..182c1c88 100644 +index 3c760ca3..3b11b965 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py -@@ -159,7 +159,7 @@ def is_manylinux1_compatible(): +@@ -178,7 +178,7 @@ def is_manylinux1_compatible(): pass # Check glibc version. CentOS 5 uses glibc 2.5. @@ -10,4 +10,13 @@ index f3b9b5b4..182c1c88 100644 + return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 5) + def is_manylinux2010_compatible(): +@@ -196,7 +196,7 @@ def is_manylinux2010_compatible(): + pass + + # Check glibc version. CentOS 6 uses glibc 2.12. +- return pip._internal.utils.glibc.have_compatible_glibc(2, 12) ++ return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 12) + + def get_darwin_arches(major, minor, machine): diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch b/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch index 99269a90..93f7ccbc 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch @@ -12,7 +12,7 @@ index 2406be21..7a87cdcf 100644 class PackageIndex(object): diff --git a/pipenv/patched/notpip/_vendor/distlib/locators.py b/pipenv/patched/notpip/_vendor/distlib/locators.py -index 11d26361..cb05b184 100644 +index 5c655c3e..a7ed9469 100644 --- a/pipenv/patched/notpip/_vendor/distlib/locators.py +++ b/pipenv/patched/notpip/_vendor/distlib/locators.py @@ -36,7 +36,7 @@ logger = logging.getLogger(__name__) @@ -33,7 +33,7 @@ index 11d26361..cb05b184 100644 is_downloadable, is_wheel, compatible, basename) def prefer_url(self, url1, url2): -@@ -1046,7 +1046,7 @@ class AggregatingLocator(Locator): +@@ -1049,7 +1049,7 @@ class AggregatingLocator(Locator): # versions which don't conform to PEP 426 / PEP 440. default_locator = AggregatingLocator( JSONLocator(), diff --git a/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch b/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch index e2f4f7de..79e12659 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/patched/notpip/_vendor/requests/packages.py b/pipenv/patched/notpip/_vendor/requests/packages.py -index 9582fa7..928d1bb 100644 +index 9582fa73..258c89ed 100644 --- a/pipenv/patched/notpip/_vendor/requests/packages.py +++ b/pipenv/patched/notpip/_vendor/requests/packages.py @@ -4,13 +4,13 @@ import sys diff --git a/tasks/vendoring/patches/patched/pip18.patch b/tasks/vendoring/patches/patched/pip19.patch similarity index 74% rename from tasks/vendoring/patches/patched/pip18.patch rename to tasks/vendoring/patches/patched/pip19.patch index f4e607c1..dc45c445 100644 --- a/tasks/vendoring/patches/patched/pip18.patch +++ b/tasks/vendoring/patches/patched/pip19.patch @@ -1,16 +1,8 @@ diff --git a/pipenv/patched/pip/_internal/download.py b/pipenv/patched/pip/_internal/download.py -index 96f3b65c..cc5b3d15 100644 +index 2bbe1762..872af328 100644 --- a/pipenv/patched/pip/_internal/download.py +++ b/pipenv/patched/pip/_internal/download.py -@@ -19,6 +19,7 @@ from pip._vendor.lockfile import LockError - from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter - from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth - from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response -+from pip._vendor.requests.sessions import Session - from pip._vendor.requests.structures import CaseInsensitiveDict - from pip._vendor.requests.utils import get_netrc_auth - # NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is -@@ -69,7 +70,7 @@ def user_agent(): +@@ -77,7 +77,7 @@ def user_agent(): Return a string representing the user agent. """ data = { @@ -19,11 +11,407 @@ index 96f3b65c..cc5b3d15 100644 "python": platform.python_version(), "implementation": { "name": platform.python_implementation(), +diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py +index 9eda3a35..67dd952c 100644 +--- a/pipenv/patched/pip/_internal/index.py ++++ b/pipenv/patched/pip/_internal/index.py +@@ -331,6 +331,9 @@ class PackageFinder(object): + # The Session we'll use to make requests + self.session = session + ++ # Kenneth's Hack ++ self.extra = None ++ + # The valid tags to check potential found wheel candidates against + self.valid_tags = get_supported( + versions=versions, +@@ -369,6 +372,23 @@ class PackageFinder(object): + ) + return "\n".join(lines) + ++ @staticmethod ++ def get_extras_links(links): ++ requires = [] ++ extras = {} ++ ++ current_list = requires ++ ++ for link in links: ++ if not link: ++ current_list = requires ++ if link.startswith('['): ++ current_list = [] ++ extras[link[1:-1]] = current_list ++ else: ++ current_list.append(link) ++ return extras ++ + @staticmethod + def _sort_locations(locations, expand_dir=False): + # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] +@@ -427,8 +447,8 @@ class PackageFinder(object): + + return files, urls + +- def _candidate_sort_key(self, candidate): +- # type: (InstallationCandidate) -> CandidateSortingKey ++ def _candidate_sort_key(self, candidate, ignore_compatibility=True): ++ # type: (InstallationCandidate, bool) -> CandidateSortingKey + """ + Function used to generate link sort key for link tuples. + The greater the return value, the more preferred it is. +@@ -448,14 +468,18 @@ class PackageFinder(object): + if candidate.location.is_wheel: + # can raise InvalidWheelFilename + wheel = Wheel(candidate.location.filename) +- if not wheel.supported(self.valid_tags): ++ if not wheel.supported(self.valid_tags) and not ignore_compatibility: + raise UnsupportedWheel( + "%s is not a supported wheel for this platform. It " + "can't be sorted." % wheel.filename + ) + if self.prefer_binary: + binary_preference = 1 +- pri = -(wheel.support_index_min(self.valid_tags)) ++ tags = self.valid_tags if not ignore_compatibility else None ++ try: ++ pri = -(wheel.support_index_min(tags=tags)) ++ except TypeError: ++ pri = -(support_num) + if wheel.build_tag is not None: + match = re.match(r'^(\d+)(.*)$', wheel.build_tag) + build_tag_groups = match.groups() +@@ -608,7 +632,10 @@ class PackageFinder(object): + + page_versions = [] + for page in self._get_pages(url_locations, project_name): +- logger.debug('Analyzing links from page %s', page.url) ++ try: ++ logger.debug('Analyzing links from page %s', page.url) ++ except AttributeError: ++ continue + with indent_log(): + page_versions.extend( + self._package_versions(page.iter_links(), search) +@@ -628,8 +655,8 @@ class PackageFinder(object): + # This is an intentional priority ordering + return file_versions + find_links_versions + page_versions + +- def find_requirement(self, req, upgrade): +- # type: (InstallRequirement, bool) -> Optional[Link] ++ def find_requirement(self, req, upgrade, ignore_compatibility=False): ++ # type: (InstallRequirement, bool, bool) -> Optional[Link] + """Try to find a Link matching req + + Expects req, an InstallRequirement and upgrade, a boolean +@@ -784,8 +811,8 @@ class PackageFinder(object): + logger.debug('Skipping link %s; %s', link, reason) + self.logged_links.add(link) + +- def _link_package_versions(self, link, search): +- # type: (Link, Search) -> Optional[InstallationCandidate] ++ def _link_package_versions(self, link, search, ignore_compatibility=True): ++ # type: (Link, Search, bool) -> Optional[InstallationCandidate] + """Return an InstallationCandidate or None""" + version = None + if link.egg_fragment: +@@ -801,12 +828,12 @@ class PackageFinder(object): + link, 'unsupported archive format: %s' % ext, + ) + return None +- if "binary" not in search.formats and ext == WHEEL_EXTENSION: ++ if "binary" not in search.formats and ext == WHEEL_EXTENSION and not ignore_compatibility: + self._log_skipped_link( + link, 'No binaries permitted for %s' % search.supplied, + ) + return None +- if "macosx10" in link.path and ext == '.zip': ++ if "macosx10" in link.path and ext == '.zip' and not ignore_compatibility: + self._log_skipped_link(link, 'macosx10 one') + return None + if ext == WHEEL_EXTENSION: +@@ -820,7 +847,7 @@ class PackageFinder(object): + link, 'wrong project name (not %s)' % search.supplied) + return None + +- if not wheel.supported(self.valid_tags): ++ if not wheel.supported(self.valid_tags) and not ignore_compatibility: + self._log_skipped_link( + link, 'it is not compatible with this Python') + return None +@@ -856,14 +883,14 @@ class PackageFinder(object): + link.filename, link.requires_python) + support_this_python = True + +- if not support_this_python: ++ if not support_this_python and not ignore_compatibility: + logger.debug("The package %s is incompatible with the python " + "version in use. Acceptable python versions are: %s", + link, link.requires_python) + return None + logger.debug('Found link %s, version: %s', link, version) + +- return InstallationCandidate(search.supplied, version, link) ++ return InstallationCandidate(search.supplied, version, link, link.requires_python) + + + def _find_name_version_sep(egg_info, canonical_name): +diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py +index 4475458a..6748957d 100644 +--- a/pipenv/patched/pip/_internal/models/candidate.py ++++ b/pipenv/patched/pip/_internal/models/candidate.py +@@ -13,11 +13,12 @@ class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + +- def __init__(self, project, version, location): +- # type: (Any, str, Link) -> None ++ def __init__(self, project, version, location, requires_python=None): ++ # type: (Any, str, Link, Any) -> None + self.project = project + self.version = parse_version(version) # type: _BaseVersion + self.location = location ++ self.requires_python = requires_python + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), +diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py +index 4f31dd5a..ed0c86b2 100644 +--- a/pipenv/patched/pip/_internal/operations/prepare.py ++++ b/pipenv/patched/pip/_internal/operations/prepare.py +@@ -17,7 +17,7 @@ from pip._internal.exceptions import ( + from pip._internal.utils.compat import expanduser + from pip._internal.utils.hashes import MissingHashes + from pip._internal.utils.logging import indent_log +-from pip._internal.utils.misc import display_path, normalize_path ++from pip._internal.utils.misc import display_path, normalize_path, rmtree + from pip._internal.utils.typing import MYPY_CHECK_RUNNING + from pip._internal.vcs import vcs + +@@ -258,14 +258,7 @@ class RequirementPreparer(object): + # package unpacked in `req.source_dir` + # package unpacked in `req.source_dir` + if os.path.exists(os.path.join(req.source_dir, 'setup.py')): +- raise PreviousBuildDirError( +- "pip can't proceed with requirements '%s' due to a" +- " pre-existing build directory (%s). This is " +- "likely due to a previous installation that failed" +- ". pip is being responsible and not assuming it " +- "can delete this. Please delete it and try again." +- % (req, req.source_dir) +- ) ++ rmtree(req.source_dir) + req.populate_link(finder, upgrade_allowed, require_hashes) + + # We can't hit this spot and have populate_link return None. +diff --git a/pipenv/patched/pip/_internal/pep425tags.py b/pipenv/patched/pip/_internal/pep425tags.py +index 1e782d1a..3c760ca3 100644 +--- a/pipenv/patched/pip/_internal/pep425tags.py ++++ b/pipenv/patched/pip/_internal/pep425tags.py +@@ -10,7 +10,10 @@ import sysconfig + import warnings + from collections import OrderedDict + +-import pip._internal.utils.glibc ++try: ++ import pip._internal.utils.glibc ++except ImportError: ++ import pip.utils.glibc + from pip._internal.utils.compat import get_extension_suffixes + from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +diff --git a/pipenv/patched/pip/_internal/req/req_install.py b/pipenv/patched/pip/_internal/req/req_install.py +index a4834b00..2c22e141 100644 +--- a/pipenv/patched/pip/_internal/req/req_install.py ++++ b/pipenv/patched/pip/_internal/req/req_install.py +@@ -588,7 +588,8 @@ class InstallRequirement(object): + self.setup_py, self.link, + ) + script = SETUPTOOLS_SHIM % self.setup_py +- base_cmd = [sys.executable, '-c', script] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ base_cmd = [sys_executable, '-c', script] + if self.isolated: + base_cmd += ["--no-user-cfg"] + egg_info_cmd = base_cmd + ['egg_info'] +@@ -746,9 +747,10 @@ class InstallRequirement(object): + with indent_log(): + # FIXME: should we do --install-headers here too? + with self.build_env: ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + call_subprocess( + [ +- sys.executable, ++ sys_executable, + '-c', + SETUPTOOLS_SHIM % self.setup_py + ] + +@@ -995,7 +997,8 @@ class InstallRequirement(object): + pycompile # type: bool + ): + # type: (...) -> List[str] +- install_args = [sys.executable, "-u"] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ install_args = [sys_executable, "-u"] + install_args.append('-c') + install_args.append(SETUPTOOLS_SHIM % self.setup_py) + install_args += list(global_options) + \ +diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py +index d1410e93..69a53bf2 100644 +--- a/pipenv/patched/pip/_internal/req/req_set.py ++++ b/pipenv/patched/pip/_internal/req/req_set.py +@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) + + class RequirementSet(object): + +- def __init__(self, require_hashes=False, check_supported_wheels=True): ++ def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): + # type: (bool, bool) -> None + """Create a RequirementSet. + """ +@@ -26,6 +26,9 @@ class RequirementSet(object): + self.requirements = OrderedDict() # type: Dict[str, InstallRequirement] # noqa: E501 + self.require_hashes = require_hashes + self.check_supported_wheels = check_supported_wheels ++ if ignore_compatibility: ++ self.check_supported_wheels = False ++ self.ignore_compatibility = (check_supported_wheels is False or ignore_compatibility is True) + + # Mapping of alias: real_name + self.requirement_aliases = {} # type: Dict[str, str] +@@ -186,7 +189,7 @@ class RequirementSet(object): + return self.requirements[name] + if name in self.requirement_aliases: + return self.requirements[self.requirement_aliases[name]] +- raise KeyError("No project with the name %r" % project_name) ++ pass + + def cleanup_files(self): + # type: () -> None +diff --git a/pipenv/patched/pip/_internal/resolve.py b/pipenv/patched/pip/_internal/resolve.py +index 33f572f1..fc8fcbe8 100644 +--- a/pipenv/patched/pip/_internal/resolve.py ++++ b/pipenv/patched/pip/_internal/resolve.py +@@ -58,7 +58,8 @@ class Resolver(object): + force_reinstall, # type: bool + isolated, # type: bool + upgrade_strategy, # type: str +- use_pep517=None # type: Optional[bool] ++ use_pep517=None, # type: Optional[bool] ++ ignore_compatibility=False, # type: bool + ): + # type: (...) -> None + super(Resolver, self).__init__() +@@ -81,8 +82,12 @@ class Resolver(object): + self.ignore_dependencies = ignore_dependencies + self.ignore_installed = ignore_installed + self.ignore_requires_python = ignore_requires_python ++ self.ignore_compatibility = ignore_compatibility + self.use_user_site = use_user_site + self.use_pep517 = use_pep517 ++ self.requires_python = None ++ if self.ignore_compatibility: ++ self.ignore_requires_python = True + + self._discovered_dependencies = \ + defaultdict(list) # type: DefaultDict[str, List] +@@ -273,7 +278,8 @@ class Resolver(object): + def _resolve_one( + self, + requirement_set, # type: RequirementSet +- req_to_install # type: InstallRequirement ++ req_to_install, # type: InstallRequirement ++ ignore_requires_python=False # type: bool + ): + # type: (...) -> List[InstallRequirement] + """Prepare a single requirements file. +@@ -298,11 +304,18 @@ class Resolver(object): + try: + check_dist_requires_python(dist) + except UnsupportedPythonVersion as err: +- if self.ignore_requires_python: ++ if self.ignore_requires_python or ignore_requires_python or self.ignore_compatibility: + logger.warning(err.args[0]) + else: + raise + ++ # A huge hack, by Kenneth Reitz. ++ try: ++ self.requires_python = check_dist_requires_python(dist, absorb=False) ++ except TypeError: ++ self.requires_python = None ++ ++ + more_reqs = [] # type: List[InstallRequirement] + + def add_req(subreq, extras_requested): +@@ -329,10 +342,14 @@ class Resolver(object): + # We add req_to_install before its dependencies, so that we + # can refer to it when adding dependencies. + if not requirement_set.has_requirement(req_to_install.name): ++ available_requested = sorted( ++ set(dist.extras) & set(req_to_install.extras) ++ ) + # 'unnamed' requirements will get added here + req_to_install.is_direct = True + requirement_set.add_requirement( + req_to_install, parent_req_name=None, ++ extras_requested=available_requested, + ) + + if not self.ignore_dependencies: +@@ -356,6 +373,20 @@ class Resolver(object): + for subreq in dist.requires(available_requested): + add_req(subreq, extras_requested=available_requested) + ++ # Hack for deep-resolving extras. ++ for available in available_requested: ++ if hasattr(dist, '_DistInfoDistribution__dep_map'): ++ for req in dist._DistInfoDistribution__dep_map[available]: ++ req = InstallRequirement( ++ str(req), ++ req_to_install, ++ isolated=self.isolated, ++ wheel_cache=self.wheel_cache, ++ use_pep517=None ++ ) ++ ++ more_reqs.append(req) ++ + if not req_to_install.editable and not req_to_install.satisfied_by: + # XXX: --no-install leads this to report 'Successfully + # downloaded' for only non-editable reqs, even though we took +diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py +index 7aaf7b5e..d56f0512 100644 +--- a/pipenv/patched/pip/_internal/utils/packaging.py ++++ b/pipenv/patched/pip/_internal/utils/packaging.py +@@ -37,7 +37,7 @@ def check_requires_python(requires_python): + requires_python_specifier = specifiers.SpecifierSet(requires_python) + + # We only use major.minor.micro +- python_version = version.parse('.'.join(map(str, sys.version_info[:3]))) ++ python_version = version.parse('{0}.{1}.{2}'.format(*sys.version_info[:3])) + return python_version in requires_python_specifier + + +@@ -57,9 +57,11 @@ def get_metadata(dist): + return feed_parser.close() + + +-def check_dist_requires_python(dist): ++def check_dist_requires_python(dist, absorb=False): + pkg_info_dict = get_metadata(dist) + requires_python = pkg_info_dict.get('Requires-Python') ++ if absorb: ++ return requires_python + try: + if not check_requires_python(requires_python): + raise exceptions.UnsupportedPythonVersion( diff --git a/pipenv/patched/pip/_internal/utils/temp_dir.py b/pipenv/patched/pip/_internal/utils/temp_dir.py -index edc506bf..84d57dac 100644 +index 2c81ad55..fbf0292a 100644 --- a/pipenv/patched/pip/_internal/utils/temp_dir.py +++ b/pipenv/patched/pip/_internal/utils/temp_dir.py -@@ -3,8 +3,10 @@ from __future__ import absolute_import +@@ -5,8 +5,10 @@ import itertools import logging import os.path import tempfile @@ -34,7 +422,7 @@ index edc506bf..84d57dac 100644 logger = logging.getLogger(__name__) -@@ -45,6 +47,20 @@ class TempDirectory(object): +@@ -47,6 +49,20 @@ class TempDirectory(object): self.path = path self.delete = delete self.kind = kind @@ -48,14 +436,14 @@ index edc506bf..84d57dac 100644 + self, + self._cleanup, + self.path, -+ warn_message=None ++ warn_message = None + ) + else: + self._finalizer = None def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.path) -@@ -72,11 +88,27 @@ class TempDirectory(object): +@@ -74,14 +90,30 @@ class TempDirectory(object): self.path = os.path.realpath( tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) ) @@ -86,424 +474,14 @@ index edc506bf..84d57dac 100644 + pass + else: + self.path = None -diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py -index 8c2f24f1..cdd48874 100644 ---- a/pipenv/patched/pip/_internal/index.py -+++ b/pipenv/patched/pip/_internal/index.py -@@ -246,6 +246,9 @@ class PackageFinder(object): - # The Session we'll use to make requests - self.session = session - -+ # Kenneth's Hack -+ self.extra = None -+ - # The valid tags to check potential found wheel candidates against - self.valid_tags = get_supported( - versions=versions, -@@ -298,6 +301,25 @@ class PackageFinder(object): - ) - self.dependency_links.extend(links) - -+ @staticmethod -+ def get_extras_links(links): -+ requires = [] -+ extras = {} -+ -+ current_list = requires -+ -+ for link in links: -+ if not link: -+ current_list = requires -+ if link.startswith('['): -+ current_list = [] -+ extras[link[1:-1]] = current_list -+ else: -+ current_list.append(link) -+ -+ return extras -+ -+ - @staticmethod - def _sort_locations(locations, expand_dir=False): - """ -@@ -350,7 +372,7 @@ class PackageFinder(object): - - return files, urls - -- def _candidate_sort_key(self, candidate): -+ def _candidate_sort_key(self, candidate, ignore_compatibility=True): - """ - Function used to generate link sort key for link tuples. - The greater the return value, the more preferred it is. -@@ -370,14 +392,19 @@ class PackageFinder(object): - if candidate.location.is_wheel: - # can raise InvalidWheelFilename - wheel = Wheel(candidate.location.filename) -- if not wheel.supported(self.valid_tags): -+ if not wheel.supported(self.valid_tags) and not ignore_compatibility: - raise UnsupportedWheel( - "%s is not a supported wheel for this platform. It " - "can't be sorted." % wheel.filename - ) - if self.prefer_binary: - binary_preference = 1 -- pri = -(wheel.support_index_min(self.valid_tags)) -+ tags = self.valid_tags if not ignore_compatibility else None -+ try: -+ pri = -(wheel.support_index_min(tags=tags)) -+ except TypeError: -+ pri = -(support_num) -+ - if wheel.build_tag is not None: - match = re.match(r'^(\d+)(.*)$', wheel.build_tag) - build_tag_groups = match.groups() -@@ -528,7 +555,10 @@ class PackageFinder(object): - - page_versions = [] - for page in self._get_pages(url_locations, project_name): -- logger.debug('Analyzing links from page %s', page.url) -+ try: -+ logger.debug('Analyzing links from page %s', page.url) -+ except AttributeError: -+ continue - with indent_log(): - page_versions.extend( - self._package_versions(page.iter_links(), search) -@@ -562,7 +592,7 @@ class PackageFinder(object): - dependency_versions - ) - -- def find_requirement(self, req, upgrade): -+ def find_requirement(self, req, upgrade, ignore_compatibility=False): - """Try to find a Link matching req - - Expects req, an InstallRequirement and upgrade, a boolean -@@ -672,7 +702,10 @@ class PackageFinder(object): - continue - seen.add(location) - -- page = self._get_page(location) -+ try: -+ page = self._get_page(location) -+ except requests.HTTPError: -+ continue - if page is None: - continue - -@@ -709,7 +742,7 @@ class PackageFinder(object): - logger.debug('Skipping link %s; %s', link, reason) - self.logged_links.add(link) - -- def _link_package_versions(self, link, search): -+ def _link_package_versions(self, link, search, ignore_compatibility=True): - """Return an InstallationCandidate or None""" - version = None - if link.egg_fragment: -@@ -725,12 +758,12 @@ class PackageFinder(object): - link, 'unsupported archive format: %s' % ext, - ) - return -- if "binary" not in search.formats and ext == wheel_ext: -+ if "binary" not in search.formats and ext == wheel_ext and not ignore_compatibility: - self._log_skipped_link( - link, 'No binaries permitted for %s' % search.supplied, - ) - return -- if "macosx10" in link.path and ext == '.zip': -+ if "macosx10" in link.path and ext == '.zip' and not ignore_compatibility: - self._log_skipped_link(link, 'macosx10 one') - return - if ext == wheel_ext: -@@ -744,7 +777,7 @@ class PackageFinder(object): - link, 'wrong project name (not %s)' % search.supplied) - return - -- if not wheel.supported(self.valid_tags): -+ if not wheel.supported(self.valid_tags) and not ignore_compatibility: - self._log_skipped_link( - link, 'it is not compatible with this Python') - return -@@ -780,14 +813,14 @@ class PackageFinder(object): - link.filename, link.requires_python) - support_this_python = True - -- if not support_this_python: -+ if not support_this_python and not ignore_compatibility: - logger.debug("The package %s is incompatible with the python" - "version in use. Acceptable python versions are:%s", - link, link.requires_python) - return - logger.debug('Found link %s, version: %s', link, version) - -- return InstallationCandidate(search.supplied, version, link) -+ return InstallationCandidate(search.supplied, version, link, link.requires_python) - - def _get_page(self, link): - return _get_html_page(link, session=self.session) -diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py -index c736de6c..a78566c1 100644 ---- a/pipenv/patched/pip/_internal/models/candidate.py -+++ b/pipenv/patched/pip/_internal/models/candidate.py -@@ -7,10 +7,11 @@ class InstallationCandidate(KeyBasedCompareMixin): - """Represents a potential "candidate" for installation. - """ - -- def __init__(self, project, version, location): -+ def __init__(self, project, version, location, requires_python=None): - self.project = project - self.version = parse_version(version) - self.location = location -+ self.requires_python = requires_python - - super(InstallationCandidate, self).__init__( - key=(self.project, self.version, self.location), -diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py -index 104bea33..ecf78b9a 100644 ---- a/pipenv/patched/pip/_internal/operations/prepare.py -+++ b/pipenv/patched/pip/_internal/operations/prepare.py -@@ -17,7 +17,7 @@ from pip._internal.exceptions import ( - from pip._internal.utils.compat import expanduser - from pip._internal.utils.hashes import MissingHashes - from pip._internal.utils.logging import indent_log --from pip._internal.utils.misc import display_path, normalize_path -+from pip._internal.utils.misc import display_path, normalize_path, rmtree - from pip._internal.vcs import vcs - - logger = logging.getLogger(__name__) -@@ -123,7 +123,11 @@ class IsSDist(DistAbstraction): - " and ".join(map(repr, sorted(missing))) - ) - -- self.req.run_egg_info() -+ try: -+ self.req.run_egg_info() -+ except (OSError, TypeError): -+ self.req._correct_build_location() -+ self.req.run_egg_info() - self.req.assert_source_matches_version() -@@ -205,16 +209,8 @@ class RequirementPreparer(object): - # installation. - # FIXME: this won't upgrade when there's an existing - # package unpacked in `req.source_dir` -- # package unpacked in `req.source_dir` - if os.path.exists(os.path.join(req.source_dir, 'setup.py')): -- raise PreviousBuildDirError( -- "pip can't proceed with requirements '%s' due to a" -- " pre-existing build directory (%s). This is " -- "likely due to a previous installation that failed" -- ". pip is being responsible and not assuming it " -- "can delete this. Please delete it and try again." -- % (req, req.source_dir) -- ) -+ rmtree(req.source_dir) - req.populate_link(finder, upgrade_allowed, require_hashes) - - # We can't hit this spot and have populate_link return None. -diff --git a/pipenv/patched/pip/_internal/pep425tags.py b/pipenv/patched/pip/_internal/pep425tags.py -index ab1a0298..763c0a24 100644 ---- a/pipenv/patched/pip/_internal/pep425tags.py -+++ b/pipenv/patched/pip/_internal/pep425tags.py -@@ -10,7 +10,11 @@ import sysconfig - import warnings - from collections import OrderedDict - --import pip._internal.utils.glibc -+try: -+ import pip._internal.utils.glibc -+except ImportError: -+ import pip.utils.glibc -+ - from pip._internal.utils.compat import get_extension_suffixes - - logger = logging.getLogger(__name__) -diff --git a/pipenv/patched/pip/_internal/req/req_install.py b/pipenv/patched/pip/_internal/req/req_install.py -index c2624fee..ee75acd6 100644 ---- a/pipenv/patched/pip/_internal/req/req_install.py -+++ b/pipenv/patched/pip/_internal/req/req_install.py -@@ -452,7 +452,8 @@ class InstallRequirement(object): - - with indent_log(): - script = SETUPTOOLS_SHIM % self.setup_py -- base_cmd = [sys.executable, '-c', script] -+ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) -+ base_cmd = [sys_executable, '-c', script] - if self.isolated: - base_cmd += ["--no-user-cfg"] - egg_info_cmd = base_cmd + ['egg_info'] -@@ -613,10 +614,11 @@ class InstallRequirement(object): - - with indent_log(): - # FIXME: should we do --install-headers here too? -+ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) - with self.build_env: - call_subprocess( - [ -- sys.executable, -+ sys_executable, - '-c', - SETUPTOOLS_SHIM % self.setup_py - ] + -@@ -834,7 +836,8 @@ class InstallRequirement(object): - - def get_install_args(self, global_options, record_filename, root, prefix, - pycompile): -- install_args = [sys.executable, "-u"] -+ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) -+ install_args = [sys_executable, "-u"] - install_args.append('-c') - install_args.append(SETUPTOOLS_SHIM % self.setup_py) - install_args += list(global_options) + \ -diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py -index b1983171..0bab231d 100644 ---- a/pipenv/patched/pip/_internal/req/req_set.py -+++ b/pipenv/patched/pip/_internal/req/req_set.py -@@ -12,13 +12,16 @@ logger = logging.getLogger(__name__) - - class RequirementSet(object): - -- def __init__(self, require_hashes=False, check_supported_wheels=True): -+ def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): - """Create a RequirementSet. - """ - - self.requirements = OrderedDict() - self.require_hashes = require_hashes - self.check_supported_wheels = check_supported_wheels -+ if ignore_compatibility: -+ self.check_supported_wheels = False -+ self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False - - # Mapping of alias: real_name - self.requirement_aliases = {} -@@ -171,7 +174,7 @@ class RequirementSet(object): - return self.requirements[name] - if name in self.requirement_aliases: - return self.requirements[self.requirement_aliases[name]] -- raise KeyError("No project with the name %r" % project_name) -+ pass - - def cleanup_files(self): - """Clean up files, remove builds.""" -diff --git a/pipenv/patched/pip/_internal/resolve.py b/pipenv/patched/pip/_internal/resolve.py -index 2d9f1c56..bedc2582 100644 ---- a/pipenv/patched/pip/_internal/resolve.py -+++ b/pipenv/patched/pip/_internal/resolve.py -@@ -35,7 +35,7 @@ class Resolver(object): - - def __init__(self, preparer, session, finder, wheel_cache, use_user_site, - ignore_dependencies, ignore_installed, ignore_requires_python, -- force_reinstall, isolated, upgrade_strategy): -+ force_reinstall, isolated, upgrade_strategy, ignore_compatibility=False): - super(Resolver, self).__init__() - assert upgrade_strategy in self._allowed_strategies - -@@ -55,7 +55,11 @@ class Resolver(object): - self.ignore_dependencies = ignore_dependencies - self.ignore_installed = ignore_installed - self.ignore_requires_python = ignore_requires_python -+ self.ignore_compatibility = ignore_compatibility - self.use_user_site = use_user_site -+ self.requires_python = None -+ if self.ignore_compatibility: -+ self.ignore_requires_python = True - - self._discovered_dependencies = defaultdict(list) - -@@ -237,7 +241,7 @@ class Resolver(object): - - return abstract_dist - -- def _resolve_one(self, requirement_set, req_to_install): -+ def _resolve_one(self, requirement_set, req_to_install, ignore_requires_python=False): - """Prepare a single requirements file. - - :return: A list of additional InstallRequirements to also install. -@@ -260,11 +264,17 @@ class Resolver(object): - try: - check_dist_requires_python(dist) - except UnsupportedPythonVersion as err: -- if self.ignore_requires_python: -+ if self.ignore_requires_python or self.ignore_compatibility: - logger.warning(err.args[0]) - else: - raise - -+ # A huge hack, by Kenneth Reitz. -+ try: -+ self.requires_python = check_dist_requires_python(dist, absorb=False) -+ except TypeError: -+ self.requires_python = None -+ - more_reqs = [] - - def add_req(subreq, extras_requested): -@@ -291,9 +301,13 @@ class Resolver(object): - # can refer to it when adding dependencies. - if not requirement_set.has_requirement(req_to_install.name): - # 'unnamed' requirements will get added here -+ available_requested = sorted( -+ set(dist.extras) & set(req_to_install.extras) -+ ) - req_to_install.is_direct = True - requirement_set.add_requirement( - req_to_install, parent_req_name=None, -+ extras_requested=available_requested, - ) - - if not self.ignore_dependencies: -@@ -317,6 +331,19 @@ class Resolver(object): - for subreq in dist.requires(available_requested): - add_req(subreq, extras_requested=available_requested) - -+ # Hack for deep-resolving extras. -+ for available in available_requested: -+ if hasattr(dist, '_DistInfoDistribution__dep_map'): -+ for req in dist._DistInfoDistribution__dep_map[available]: -+ req = install_req_from_req( -+ str(req), -+ req_to_install, -+ isolated=self.isolated, -+ wheel_cache=self.wheel_cache, -+ ) -+ -+ more_reqs.append(req) -+ - if not req_to_install.editable and not req_to_install.satisfied_by: - # XXX: --no-install leads this to report 'Successfully - # downloaded' for only non-editable reqs, even though we took -diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py -index c43142f0..f241cce0 100644 ---- a/pipenv/patched/pip/_internal/utils/packaging.py -+++ b/pipenv/patched/pip/_internal/utils/packaging.py -@@ -29,7 +29,7 @@ def check_requires_python(requires_python): - requires_python_specifier = specifiers.SpecifierSet(requires_python) - - # We only use major.minor.micro -- python_version = version.parse('.'.join(map(str, sys.version_info[:3]))) -+ python_version = version.parse('{0}.{1}.{2}'.format(*sys.version_info[:3])) - return python_version in requires_python_specifier - - -@@ -48,9 +48,11 @@ def get_metadata(dist): - return feed_parser.close() - - --def check_dist_requires_python(dist): -+def check_dist_requires_python(dist, absorb=True): - pkg_info_dict = get_metadata(dist) - requires_python = pkg_info_dict.get('Requires-Python') -+ if absorb: -+ return requires_python - try: - if not check_requires_python(requires_python): - raise exceptions.UnsupportedPythonVersion( + class AdjacentTempDirectory(TempDirectory): diff --git a/pipenv/patched/pip/_internal/wheel.py b/pipenv/patched/pip/_internal/wheel.py -index 5ce890eb..46c0181c 100644 +index 67bcc7f7..968cdff9 100644 --- a/pipenv/patched/pip/_internal/wheel.py +++ b/pipenv/patched/pip/_internal/wheel.py -@@ -83,7 +83,7 @@ def fix_script(path): +@@ -114,7 +114,7 @@ def fix_script(path): firstline = script.readline() if not firstline.startswith(b'#!python'): return False @@ -512,7 +490,7 @@ index 5ce890eb..46c0181c 100644 firstline = b'#!' + exename + os.linesep.encode("ascii") rest = script.read() with open(path, 'wb') as script: -@@ -167,7 +167,8 @@ def message_about_scripts_not_on_PATH(scripts): +@@ -201,7 +201,8 @@ def message_about_scripts_not_on_PATH(scripts): ] # If an executable sits with sys.executable, we don't warn for it. # This covers the case of venv invocations without activating the venv. @@ -522,7 +500,7 @@ index 5ce890eb..46c0181c 100644 warn_for = { parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() if os.path.normcase(parent_dir) not in not_warn_dirs -@@ -667,8 +668,9 @@ class WheelBuilder(object): +@@ -901,8 +902,9 @@ class WheelBuilder(object): # isolating. Currently, it breaks Python in virtualenvs, because it # relies on site.py to find parts of the standard library outside the # virtualenv. From 52cf5e49606378eddf9ba431937e6e01cbba8ebd Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 19 Mar 2019 09:33:11 -0400 Subject: [PATCH 02/22] Update to pip 19.0.3 Signed-off-by: Dan Ryan --- pipenv/patched/notpip/COPYING | 14 + pipenv/patched/notpip/LICENSE.BSD | 23 + pipenv/patched/notpip/__init__.py | 2 +- pipenv/patched/notpip/_internal/build_env.py | 183 +- pipenv/patched/notpip/_internal/cache.py | 22 + .../notpip/_internal/cli/base_command.py | 89 +- .../notpip/_internal/cli/cmdoptions.py | 201 +- .../notpip/_internal/cli/main_parser.py | 12 +- .../notpip/_internal/commands/check.py | 4 +- .../notpip/_internal/commands/download.py | 2 + .../notpip/_internal/commands/install.py | 67 +- .../patched/notpip/_internal/commands/list.py | 19 +- .../notpip/_internal/commands/wheel.py | 7 +- pipenv/patched/notpip/_internal/download.py | 101 +- pipenv/patched/notpip/_internal/exceptions.py | 8 +- pipenv/patched/notpip/_internal/index.py | 469 ++-- pipenv/patched/notpip/_internal/locations.py | 25 +- .../notpip/_internal/models/candidate.py | 10 +- .../notpip/_internal/models/format_control.py | 25 +- .../patched/notpip/_internal/models/index.py | 2 + .../patched/notpip/_internal/models/link.py | 32 +- .../notpip/_internal/operations/check.py | 25 +- .../notpip/_internal/operations/freeze.py | 221 +- .../notpip/_internal/operations/prepare.py | 119 +- pipenv/patched/notpip/_internal/pep425tags.py | 91 +- pipenv/patched/notpip/_internal/pyproject.py | 67 +- .../patched/notpip/_internal/req/__init__.py | 12 +- .../notpip/_internal/req/constructors.py | 107 +- .../patched/notpip/_internal/req/req_file.py | 86 +- .../notpip/_internal/req/req_install.py | 345 ++- .../patched/notpip/_internal/req/req_set.py | 32 +- .../notpip/_internal/req/req_tracker.py | 14 +- .../notpip/_internal/req/req_uninstall.py | 182 +- pipenv/patched/notpip/_internal/resolve.py | 74 +- .../patched/notpip/_internal/utils/appdirs.py | 12 + .../patched/notpip/_internal/utils/compat.py | 32 +- .../notpip/_internal/utils/deprecation.py | 1 + .../notpip/_internal/utils/encoding.py | 8 +- .../notpip/_internal/utils/filesystem.py | 2 + .../patched/notpip/_internal/utils/glibc.py | 11 +- .../patched/notpip/_internal/utils/hashes.py | 21 + .../patched/notpip/_internal/utils/logging.py | 99 +- pipenv/patched/notpip/_internal/utils/misc.py | 178 +- .../notpip/_internal/utils/outdated.py | 14 +- .../notpip/_internal/utils/packaging.py | 14 +- .../notpip/_internal/utils/temp_dir.py | 77 +- pipenv/patched/notpip/_internal/utils/ui.py | 32 +- .../patched/notpip/_internal/vcs/__init__.py | 131 +- pipenv/patched/notpip/_internal/vcs/bazaar.py | 22 +- pipenv/patched/notpip/_internal/vcs/git.py | 87 +- .../patched/notpip/_internal/vcs/mercurial.py | 28 +- .../notpip/_internal/vcs/subversion.py | 49 +- pipenv/patched/notpip/_internal/wheel.py | 440 +++- pipenv/patched/notpip/_vendor/__init__.py | 1 + .../notpip/_vendor/certifi/__init__.py | 4 +- .../patched/notpip/_vendor/certifi/cacert.pem | 272 +- pipenv/patched/notpip/_vendor/certifi/core.py | 17 - .../notpip/_vendor/colorama/__init__.py | 3 +- .../notpip/_vendor/colorama/ansitowin32.py | 43 +- .../notpip/_vendor/colorama/initialise.py | 2 - .../patched/notpip/_vendor/colorama/win32.py | 18 +- .../notpip/_vendor/colorama/winterm.py | 11 +- .../notpip/_vendor/distlib/__init__.py | 2 +- .../notpip/_vendor/distlib/database.py | 7 +- .../notpip/_vendor/distlib/locators.py | 11 +- .../notpip/_vendor/distlib/metadata.py | 9 +- .../patched/notpip/_vendor/distlib/scripts.py | 6 +- pipenv/patched/notpip/_vendor/distlib/util.py | 15 +- .../patched/notpip/_vendor/distlib/wheel.py | 8 +- pipenv/patched/notpip/_vendor/idna/core.py | 5 +- .../patched/notpip/_vendor/idna/idnadata.py | 122 +- .../notpip/_vendor/idna/package_data.py | 2 +- .../patched/notpip/_vendor/idna/uts46data.py | 660 ++--- .../notpip/_vendor/packaging/__about__.py | 14 +- .../notpip/_vendor/packaging/__init__.py | 20 +- .../notpip/_vendor/packaging/_compat.py | 7 +- .../notpip/_vendor/packaging/_structures.py | 2 - .../notpip/_vendor/packaging/markers.py | 89 +- .../notpip/_vendor/packaging/requirements.py | 34 +- .../notpip/_vendor/packaging/specifiers.py | 67 +- .../patched/notpip/_vendor/packaging/utils.py | 8 +- .../notpip/_vendor/packaging/version.py | 49 +- .../patched/notpip/_vendor/pep517/__init__.py | 2 +- .../notpip/_vendor/pep517/_in_process.py | 47 +- pipenv/patched/notpip/_vendor/pep517/build.py | 108 + pipenv/patched/notpip/_vendor/pep517/check.py | 32 +- .../patched/notpip/_vendor/pep517/colorlog.py | 21 +- .../patched/notpip/_vendor/pep517/envbuild.py | 14 +- .../patched/notpip/_vendor/pep517/wrappers.py | 47 +- .../notpip/_vendor/pkg_resources/__init__.py | 26 +- pipenv/patched/notpip/_vendor/pyparsing.py | 2342 +++++++++++------ .../patched/notpip/_vendor/pytoml/__init__.py | 3 +- .../patched/notpip/_vendor/pytoml/parser.py | 55 +- pipenv/patched/notpip/_vendor/pytoml/test.py | 30 + pipenv/patched/notpip/_vendor/pytoml/utils.py | 67 + .../patched/notpip/_vendor/pytoml/writer.py | 39 +- .../notpip/_vendor/requests/__init__.py | 17 +- .../notpip/_vendor/requests/__version__.py | 4 +- .../notpip/_vendor/requests/adapters.py | 27 +- pipenv/patched/notpip/_vendor/requests/api.py | 20 +- .../patched/notpip/_vendor/requests/auth.py | 4 +- .../patched/notpip/_vendor/requests/compat.py | 3 +- .../notpip/_vendor/requests/cookies.py | 31 +- .../patched/notpip/_vendor/requests/help.py | 3 +- .../patched/notpip/_vendor/requests/hooks.py | 4 +- .../patched/notpip/_vendor/requests/models.py | 19 +- .../notpip/_vendor/requests/sessions.py | 63 +- .../notpip/_vendor/requests/status_codes.py | 2 +- .../patched/notpip/_vendor/requests/utils.py | 23 +- pipenv/patched/notpip/_vendor/six.py | 65 +- .../notpip/_vendor/urllib3/__init__.py | 9 +- .../notpip/_vendor/urllib3/_collections.py | 5 +- .../notpip/_vendor/urllib3/connection.py | 40 +- .../notpip/_vendor/urllib3/connectionpool.py | 18 +- .../urllib3/contrib/_appengine_environ.py | 30 + .../_vendor/urllib3/contrib/appengine.py | 36 +- .../_vendor/urllib3/contrib/ntlmpool.py | 3 +- .../_vendor/urllib3/contrib/pyopenssl.py | 25 +- .../urllib3/packages/backports/makefile.py | 2 +- .../_vendor/urllib3/packages/ordered_dict.py | 259 -- .../ssl_match_hostname/_implementation.py | 3 +- .../notpip/_vendor/urllib3/poolmanager.py | 1 + .../patched/notpip/_vendor/urllib3/request.py | 2 +- .../notpip/_vendor/urllib3/response.py | 47 +- .../notpip/_vendor/urllib3/util/connection.py | 8 + .../notpip/_vendor/urllib3/util/response.py | 10 +- .../notpip/_vendor/urllib3/util/retry.py | 2 +- .../notpip/_vendor/urllib3/util/ssl_.py | 35 +- .../notpip/_vendor/urllib3/util/wait.py | 5 +- pipenv/patched/notpip/appdirs.LICENSE.txt | 23 + pipenv/patched/notpip/distro.LICENSE | 202 ++ pipenv/patched/notpip/ipaddress.LICENSE | 50 + .../patched/notpip/packaging.LICENSE.APACHE | 177 ++ pipenv/patched/notpip/pyparsing.LICENSE | 18 + pipenv/patched/notpip/retrying.LICENSE | 202 ++ pipenv/patched/notpip/six.LICENSE | 18 + pipenv/patched/notpip/urllib3.LICENSE | 19 + pipenv/patched/notpip/webencodings.LICENSE | 31 + 138 files changed, 6760 insertions(+), 3096 deletions(-) create mode 100644 pipenv/patched/notpip/COPYING create mode 100644 pipenv/patched/notpip/LICENSE.BSD create mode 100644 pipenv/patched/notpip/_vendor/pep517/build.py create mode 100644 pipenv/patched/notpip/_vendor/pytoml/test.py create mode 100644 pipenv/patched/notpip/_vendor/pytoml/utils.py create mode 100644 pipenv/patched/notpip/_vendor/urllib3/contrib/_appengine_environ.py delete mode 100644 pipenv/patched/notpip/_vendor/urllib3/packages/ordered_dict.py create mode 100644 pipenv/patched/notpip/appdirs.LICENSE.txt create mode 100644 pipenv/patched/notpip/distro.LICENSE create mode 100644 pipenv/patched/notpip/ipaddress.LICENSE create mode 100644 pipenv/patched/notpip/packaging.LICENSE.APACHE create mode 100644 pipenv/patched/notpip/pyparsing.LICENSE create mode 100644 pipenv/patched/notpip/retrying.LICENSE create mode 100644 pipenv/patched/notpip/six.LICENSE create mode 100644 pipenv/patched/notpip/urllib3.LICENSE create mode 100644 pipenv/patched/notpip/webencodings.LICENSE diff --git a/pipenv/patched/notpip/COPYING b/pipenv/patched/notpip/COPYING new file mode 100644 index 00000000..f067af3a --- /dev/null +++ b/pipenv/patched/notpip/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008-2011 INADA Naoki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/pipenv/patched/notpip/LICENSE.BSD b/pipenv/patched/notpip/LICENSE.BSD new file mode 100644 index 00000000..42ce7b75 --- /dev/null +++ b/pipenv/patched/notpip/LICENSE.BSD @@ -0,0 +1,23 @@ +Copyright (c) Donald Stufft and individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/patched/notpip/__init__.py b/pipenv/patched/notpip/__init__.py index ae265fa7..f48c1ca6 100644 --- a/pipenv/patched/notpip/__init__.py +++ b/pipenv/patched/notpip/__init__.py @@ -1 +1 @@ -__version__ = "18.1" +__version__ = "19.0.3" diff --git a/pipenv/patched/notpip/_internal/build_env.py b/pipenv/patched/notpip/_internal/build_env.py index 6d696fbd..d38adc49 100644 --- a/pipenv/patched/notpip/_internal/build_env.py +++ b/pipenv/patched/notpip/_internal/build_env.py @@ -4,98 +4,173 @@ import logging import os import sys +import textwrap +from collections import OrderedDict from distutils.sysconfig import get_python_lib from sysconfig import get_paths from pipenv.patched.notpip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet +from pipenv.patched.notpip import __file__ as pip_location from pipenv.patched.notpip._internal.utils.misc import call_subprocess from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import open_spinner +if MYPY_CHECK_RUNNING: + from typing import Tuple, Set, Iterable, Optional, List # noqa: F401 + from pipenv.patched.notpip._internal.index import PackageFinder # noqa: F401 + logger = logging.getLogger(__name__) +class _Prefix: + + def __init__(self, path): + # type: (str) -> None + self.path = path + self.setup = False + self.bin_dir = get_paths( + 'nt' if os.name == 'nt' else 'posix_prefix', + vars={'base': path, 'platbase': path} + )['scripts'] + # Note: prefer distutils' sysconfig to get the + # library paths so PyPy is correctly supported. + purelib = get_python_lib(plat_specific=False, prefix=path) + platlib = get_python_lib(plat_specific=True, prefix=path) + if purelib == platlib: + self.lib_dirs = [purelib] + else: + self.lib_dirs = [purelib, platlib] + + class BuildEnvironment(object): """Creates and manages an isolated environment to install build deps """ def __init__(self): + # type: () -> None self._temp_dir = TempDirectory(kind="build-env") self._temp_dir.create() - @property - def path(self): - return self._temp_dir.path + self._prefixes = OrderedDict(( + (name, _Prefix(os.path.join(self._temp_dir.path, name))) + for name in ('normal', 'overlay') + )) + + self._bin_dirs = [] # type: List[str] + self._lib_dirs = [] # type: List[str] + for prefix in reversed(list(self._prefixes.values())): + self._bin_dirs.append(prefix.bin_dir) + self._lib_dirs.extend(prefix.lib_dirs) + + # Customize site to: + # - ensure .pth files are honored + # - prevent access to system site packages + system_sites = { + os.path.normcase(site) for site in ( + get_python_lib(plat_specific=False), + get_python_lib(plat_specific=True), + ) + } + self._site_dir = os.path.join(self._temp_dir.path, 'site') + if not os.path.exists(self._site_dir): + os.mkdir(self._site_dir) + with open(os.path.join(self._site_dir, 'sitecustomize.py'), 'w') as fp: + fp.write(textwrap.dedent( + ''' + import os, site, sys + + # First, drop system-sites related paths. + original_sys_path = sys.path[:] + known_paths = set() + for path in {system_sites!r}: + site.addsitedir(path, known_paths=known_paths) + system_paths = set( + os.path.normcase(path) + for path in sys.path[len(original_sys_path):] + ) + original_sys_path = [ + path for path in original_sys_path + if os.path.normcase(path) not in system_paths + ] + sys.path = original_sys_path + + # Second, add lib directories. + # ensuring .pth file are processed. + for path in {lib_dirs!r}: + assert not path in sys.path + site.addsitedir(path) + ''' + ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)) def __enter__(self): - self.save_path = os.environ.get('PATH', None) - self.save_pythonpath = os.environ.get('PYTHONPATH', None) - self.save_nousersite = os.environ.get('PYTHONNOUSERSITE', None) + self._save_env = { + name: os.environ.get(name, None) + for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH') + } - install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' - install_dirs = get_paths(install_scheme, vars={ - 'base': self.path, - 'platbase': self.path, + path = self._bin_dirs[:] + old_path = self._save_env['PATH'] + if old_path: + path.extend(old_path.split(os.pathsep)) + + pythonpath = [self._site_dir] + + os.environ.update({ + 'PATH': os.pathsep.join(path), + 'PYTHONNOUSERSITE': '1', + 'PYTHONPATH': os.pathsep.join(pythonpath), }) - scripts = install_dirs['scripts'] - if self.save_path: - os.environ['PATH'] = scripts + os.pathsep + self.save_path - else: - os.environ['PATH'] = scripts + os.pathsep + os.defpath - - # Note: prefer distutils' sysconfig to get the - # library paths so PyPy is correctly supported. - purelib = get_python_lib(plat_specific=0, prefix=self.path) - platlib = get_python_lib(plat_specific=1, prefix=self.path) - if purelib == platlib: - lib_dirs = purelib - else: - lib_dirs = purelib + os.pathsep + platlib - if self.save_pythonpath: - os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ - self.save_pythonpath - else: - os.environ['PYTHONPATH'] = lib_dirs - - os.environ['PYTHONNOUSERSITE'] = '1' - - return self.path - def __exit__(self, exc_type, exc_val, exc_tb): - def restore_var(varname, old_value): + for varname, old_value in self._save_env.items(): if old_value is None: os.environ.pop(varname, None) else: os.environ[varname] = old_value - restore_var('PATH', self.save_path) - restore_var('PYTHONPATH', self.save_pythonpath) - restore_var('PYTHONNOUSERSITE', self.save_nousersite) - def cleanup(self): + # type: () -> None self._temp_dir.cleanup() - def missing_requirements(self, reqs): - """Return a list of the requirements from reqs that are not present + def check_requirements(self, reqs): + # type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]] + """Return 2 sets: + - conflicting requirements: set of (installed, wanted) reqs tuples + - missing requirements: set of reqs """ - missing = [] - with self: - ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep)) + missing = set() + conflicting = set() + if reqs: + ws = WorkingSet(self._lib_dirs) for req in reqs: try: if ws.find(Requirement.parse(req)) is None: - missing.append(req) - except VersionConflict: - missing.append(req) - return missing + missing.add(req) + except VersionConflict as e: + conflicting.add((str(e.args[0].as_requirement()), + str(e.args[1]))) + return conflicting, missing - def install_requirements(self, finder, requirements, message): + def install_requirements( + self, + finder, # type: PackageFinder + requirements, # type: Iterable[str] + prefix_as_string, # type: str + message # type: Optional[str] + ): + # type: (...) -> None + prefix = self._prefixes[prefix_as_string] + assert not prefix.setup + prefix.setup = True + if not requirements: + return args = [ - sys.executable, '-m', 'pip', 'install', '--ignore-installed', - '--no-user', '--prefix', self.path, '--no-warn-script-location', - ] + sys.executable, os.path.dirname(pip_location), 'install', + '--ignore-installed', '--no-user', '--prefix', prefix.path, + '--no-warn-script-location', + ] # type: List[str] if logger.getEffectiveLevel() <= logging.DEBUG: args.append('-v') for format_control in ('no_binary', 'only_binary'): @@ -114,8 +189,6 @@ class BuildEnvironment(object): args.extend(['--trusted-host', host]) if finder.allow_all_prereleases: args.append('--pre') - if finder.process_dependency_links: - args.append('--process-dependency-links') args.append('--') args.extend(requirements) with open_spinner(message) as spinner: @@ -138,5 +211,5 @@ class NoOpBuildEnvironment(BuildEnvironment): def cleanup(self): pass - def install_requirements(self, finder, requirements, message): + def install_requirements(self, finder, requirements, prefix, message): raise NotImplementedError() diff --git a/pipenv/patched/notpip/_internal/cache.py b/pipenv/patched/notpip/_internal/cache.py index d91b8170..9f35e83d 100644 --- a/pipenv/patched/notpip/_internal/cache.py +++ b/pipenv/patched/notpip/_internal/cache.py @@ -12,8 +12,13 @@ from pipenv.patched.notpip._internal.download import path_to_url from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.wheel import InvalidWheelFilename, Wheel +if MYPY_CHECK_RUNNING: + from typing import Optional, Set, List, Any # noqa: F401 + from pipenv.patched.notpip._internal.index import FormatControl # noqa: F401 + logger = logging.getLogger(__name__) @@ -29,6 +34,7 @@ class Cache(object): """ def __init__(self, cache_dir, format_control, allowed_formats): + # type: (str, FormatControl, Set[str]) -> None super(Cache, self).__init__() self.cache_dir = expanduser(cache_dir) if cache_dir else None self.format_control = format_control @@ -38,6 +44,7 @@ class Cache(object): assert self.allowed_formats.union(_valid_formats) == _valid_formats def _get_cache_path_parts(self, link): + # type: (Link) -> List[str] """Get parts of part that must be os.path.joined with cache_dir """ @@ -63,6 +70,7 @@ class Cache(object): return parts def _get_candidates(self, link, package_name): + # type: (Link, Optional[str]) -> List[Any] can_not_cache = ( not self.cache_dir or not package_name or @@ -87,23 +95,27 @@ class Cache(object): raise def get_path_for_link(self, link): + # type: (Link) -> str """Return a directory to store cached items in for link. """ raise NotImplementedError() def get(self, link, package_name): + # type: (Link, Optional[str]) -> Link """Returns a link to a cached item if it exists, otherwise returns the passed link. """ raise NotImplementedError() def _link_for_candidate(self, link, candidate): + # type: (Link, str) -> Link root = self.get_path_for_link(link) path = os.path.join(root, candidate) return Link(path_to_url(path)) def cleanup(self): + # type: () -> None pass @@ -112,11 +124,13 @@ class SimpleWheelCache(Cache): """ def __init__(self, cache_dir, format_control): + # type: (str, FormatControl) -> None super(SimpleWheelCache, self).__init__( cache_dir, format_control, {"binary"} ) def get_path_for_link(self, link): + # type: (Link) -> str """Return a directory to store cached wheels for link Because there are M wheels for any one sdist, we provide a directory @@ -137,6 +151,7 @@ class SimpleWheelCache(Cache): return os.path.join(self.cache_dir, "wheels", *parts) def get(self, link, package_name): + # type: (Link, Optional[str]) -> Link candidates = [] for wheel_name in self._get_candidates(link, package_name): @@ -160,6 +175,7 @@ class EphemWheelCache(SimpleWheelCache): """ def __init__(self, format_control): + # type: (FormatControl) -> None self._temp_dir = TempDirectory(kind="ephem-wheel-cache") self._temp_dir.create() @@ -168,6 +184,7 @@ class EphemWheelCache(SimpleWheelCache): ) def cleanup(self): + # type: () -> None self._temp_dir.cleanup() @@ -179,6 +196,7 @@ class WheelCache(Cache): """ def __init__(self, cache_dir, format_control): + # type: (str, FormatControl) -> None super(WheelCache, self).__init__( cache_dir, format_control, {'binary'} ) @@ -186,17 +204,21 @@ class WheelCache(Cache): self._ephem_cache = EphemWheelCache(format_control) def get_path_for_link(self, link): + # type: (Link) -> str return self._wheel_cache.get_path_for_link(link) def get_ephem_path_for_link(self, link): + # type: (Link) -> str return self._ephem_cache.get_path_for_link(link) def get(self, link, package_name): + # type: (Link, Optional[str]) -> Link retval = self._wheel_cache.get(link, package_name) if retval is link: retval = self._ephem_cache.get(link, package_name) return retval def cleanup(self): + # type: () -> None self._wheel_cache.cleanup() self._ephem_cache.cleanup() diff --git a/pipenv/patched/notpip/_internal/cli/base_command.py b/pipenv/patched/notpip/_internal/cli/base_command.py index 229831f2..4aa16da6 100644 --- a/pipenv/patched/notpip/_internal/cli/base_command.py +++ b/pipenv/patched/notpip/_internal/cli/base_command.py @@ -1,11 +1,13 @@ """Base Command class, and related routines""" -from __future__ import absolute_import +from __future__ import absolute_import, print_function import logging import logging.config import optparse import os +import platform import sys +import traceback from pipenv.patched.notpip._internal.cli import cmdoptions from pipenv.patched.notpip._internal.cli.parser import ( @@ -26,13 +28,19 @@ from pipenv.patched.notpip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) from pipenv.patched.notpip._internal.req.req_file import parse_requirements -from pipenv.patched.notpip._internal.utils.logging import setup_logging -from pipenv.patched.notpip._internal.utils.misc import get_prog, normalize_path +from pipenv.patched.notpip._internal.utils.deprecation import deprecated +from pipenv.patched.notpip._internal.utils.logging import BrokenStdoutLoggingError, setup_logging +from pipenv.patched.notpip._internal.utils.misc import ( + get_prog, normalize_path, redact_password_from_url, +) from pipenv.patched.notpip._internal.utils.outdated import pip_version_check from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional # noqa: F401 + from typing import Optional, List, Tuple, Any # noqa: F401 + from optparse import Values # noqa: F401 + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + from pipenv.patched.notpip._internal.req.req_set import RequirementSet # noqa: F401 __all__ = ['Command'] @@ -46,6 +54,7 @@ class Command(object): ignore_require_venv = False # type: bool def __init__(self, isolated=False): + # type: (bool) -> None parser_kw = { 'usage': self.usage, 'prog': '%s %s' % (get_prog(), self.name), @@ -69,7 +78,12 @@ class Command(object): ) self.parser.add_option_group(gen_opts) + def run(self, options, args): + # type: (Values, List[Any]) -> Any + raise NotImplementedError + def _build_session(self, options, retries=None, timeout=None): + # type: (Values, Optional[int], Optional[int]) -> PipSession session = PipSession( cache=( normalize_path(os.path.join(options.cache_dir, "http")) @@ -106,21 +120,43 @@ class Command(object): return session def parse_args(self, args): + # type: (List[str]) -> Tuple # factored out for testability return self.parser.parse_args(args) def main(self, args): + # type: (List[str]) -> int options, args = self.parse_args(args) # Set verbosity so that it can be used elsewhere. self.verbosity = options.verbose - options.quiet - setup_logging( + level_number = setup_logging( verbosity=self.verbosity, no_color=options.no_color, user_log_file=options.log, ) + if sys.version_info[:2] == (3, 4): + deprecated( + "Python 3.4 support has been deprecated. pip 19.1 will be the " + "last one supporting it. Please upgrade your Python as Python " + "3.4 won't be maintained after March 2019 (cf PEP 429).", + replacement=None, + gone_in='19.2', + ) + elif sys.version_info[:2] == (2, 7): + message = ( + "A future version of pip will drop support for Python 2.7." + ) + if platform.python_implementation() == "CPython": + message = ( + "Python 2.7 will reach the end of its life on January " + "1st, 2020. Please upgrade your Python as Python 2.7 " + "won't be maintained after that date. " + ) + message + deprecated(message, replacement=None, gone_in=None) + # TODO: Try to get these passing down from the command? # without resorting to os.environ to hold these. # This also affects isolated builds and it should. @@ -159,6 +195,14 @@ class Command(object): logger.critical('ERROR: %s', exc) logger.debug('Exception information:', exc_info=True) + return ERROR + except BrokenStdoutLoggingError: + # Bypass our logger and write any remaining messages to stderr + # because stdout no longer works. + print('ERROR: Pipe to stdout was broken', file=sys.stderr) + if level_number <= logging.DEBUG: + traceback.print_exc(file=sys.stderr) + return ERROR except KeyboardInterrupt: logger.critical('Operation cancelled by user') @@ -195,8 +239,15 @@ class Command(object): class RequirementCommand(Command): @staticmethod - def populate_requirement_set(requirement_set, args, options, finder, - session, name, wheel_cache): + def populate_requirement_set(requirement_set, # type: RequirementSet + args, # type: List[str] + options, # type: Values + finder, # type: PackageFinder + session, # type: PipSession + name, # type: str + wheel_cache # type: Optional[WheelCache] + ): + # type: (...) -> None """ Marshal cmd line args into a requirement set. """ @@ -214,6 +265,7 @@ class RequirementCommand(Command): for req in args: req_to_add = install_req_from_line( req, None, isolated=options.isolated_mode, + use_pep517=options.use_pep517, wheel_cache=wheel_cache ) req_to_add.is_direct = True @@ -223,6 +275,7 @@ class RequirementCommand(Command): req_to_add = install_req_from_editable( req, isolated=options.isolated_mode, + use_pep517=options.use_pep517, wheel_cache=wheel_cache ) req_to_add.is_direct = True @@ -232,7 +285,8 @@ class RequirementCommand(Command): for req_to_add in parse_requirements( filename, finder=finder, options=options, session=session, - wheel_cache=wheel_cache): + wheel_cache=wheel_cache, + use_pep517=options.use_pep517): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) # If --require-hashes was a line in a requirements file, tell @@ -251,15 +305,25 @@ class RequirementCommand(Command): 'You must give at least one requirement to %(name)s ' '(see "pip help %(name)s")' % opts) - def _build_package_finder(self, options, session, - platform=None, python_versions=None, - abi=None, implementation=None): + def _build_package_finder( + self, + options, # type: Values + session, # type: PipSession + platform=None, # type: Optional[str] + python_versions=None, # type: Optional[List[str]] + abi=None, # type: Optional[str] + implementation=None # type: Optional[str] + ): + # type: (...) -> PackageFinder """ Create a package finder appropriate to this requirement command. """ index_urls = [options.index_url] + options.extra_index_urls if options.no_index: - logger.debug('Ignoring indexes: %s', ','.join(index_urls)) + logger.debug( + 'Ignoring indexes: %s', + ','.join(redact_password_from_url(url) for url in index_urls), + ) index_urls = [] return PackageFinder( @@ -268,7 +332,6 @@ class RequirementCommand(Command): index_urls=index_urls, trusted_hosts=options.trusted_hosts, allow_all_prereleases=options.pre, - process_dependency_links=options.process_dependency_links, session=session, platform=platform, versions=python_versions, diff --git a/pipenv/patched/notpip/_internal/cli/cmdoptions.py b/pipenv/patched/notpip/_internal/cli/cmdoptions.py index a075a67e..3c5a7084 100644 --- a/pipenv/patched/notpip/_internal/cli/cmdoptions.py +++ b/pipenv/patched/notpip/_internal/cli/cmdoptions.py @@ -9,7 +9,9 @@ pass on state. To be consistent, all options will follow this design. """ from __future__ import absolute_import +import textwrap import warnings +from distutils.util import strtobool from functools import partial from optparse import SUPPRESS_HELP, Option, OptionGroup @@ -22,10 +24,27 @@ from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import BAR_TYPES if MYPY_CHECK_RUNNING: - from typing import Any # noqa: F401 + from typing import Any, Callable, Dict, List, Optional, Union # noqa: F401 + from optparse import OptionParser, Values # noqa: F401 + from pipenv.patched.notpip._internal.cli.parser import ConfigOptionParser # noqa: F401 + + +def raise_option_error(parser, option, msg): + """ + Raise an option parsing error using parser.error(). + + Args: + parser: an OptionParser instance. + option: an Option instance. + msg: the error text. + """ + msg = '{} error: {}'.format(option, msg) + msg = textwrap.fill(' '.join(msg.split())) + parser.error(msg) def make_option_group(group, parser): + # type: (Dict[str, Any], ConfigOptionParser) -> OptionGroup """ Return an OptionGroup object group -- assumed to be dict with 'name' and 'options' keys @@ -38,6 +57,7 @@ def make_option_group(group, parser): def check_install_build_global(options, check_options=None): + # type: (Values, Optional[Values]) -> None """Disable wheels if per-setup.py call options are set. :param options: The OptionParser options to update. @@ -60,6 +80,7 @@ def check_install_build_global(options, check_options=None): def check_dist_restriction(options, check_target=False): + # type: (Values, bool) -> None """Function for determining if custom platform options are allowed. :param options: The OptionParser options. @@ -108,7 +129,7 @@ help_ = partial( dest='help', action='help', help='Show help.', -) # type: Any +) # type: Callable[..., Option] isolated_mode = partial( Option, @@ -120,7 +141,7 @@ isolated_mode = partial( "Run pip in an isolated mode, ignoring environment variables and user " "configuration." ), -) +) # type: Callable[..., Option] require_virtualenv = partial( Option, @@ -130,7 +151,7 @@ require_virtualenv = partial( action='store_true', default=False, help=SUPPRESS_HELP -) # type: Any +) # type: Callable[..., Option] verbose = partial( Option, @@ -139,7 +160,7 @@ verbose = partial( action='count', default=0, help='Give more output. Option is additive, and can be used up to 3 times.' -) +) # type: Callable[..., Option] no_color = partial( Option, @@ -148,7 +169,7 @@ no_color = partial( action='store_true', default=False, help="Suppress colored output", -) +) # type: Callable[..., Option] version = partial( Option, @@ -156,7 +177,7 @@ version = partial( dest='version', action='store_true', help='Show version and exit.', -) # type: Any +) # type: Callable[..., Option] quiet = partial( Option, @@ -169,7 +190,7 @@ quiet = partial( ' times (corresponding to WARNING, ERROR, and CRITICAL logging' ' levels).' ), -) # type: Any +) # type: Callable[..., Option] progress_bar = partial( Option, @@ -182,7 +203,7 @@ progress_bar = partial( 'Specify type of progress to be displayed [' + '|'.join(BAR_TYPES.keys()) + '] (default: %default)' ), -) # type: Any +) # type: Callable[..., Option] log = partial( Option, @@ -190,7 +211,7 @@ log = partial( dest="log", metavar="path", help="Path to a verbose appending log." -) # type: Any +) # type: Callable[..., Option] no_input = partial( Option, @@ -200,7 +221,7 @@ no_input = partial( action='store_true', default=False, help=SUPPRESS_HELP -) # type: Any +) # type: Callable[..., Option] proxy = partial( Option, @@ -209,7 +230,7 @@ proxy = partial( type='str', default='', help="Specify a proxy in the form [user:passwd@]proxy.server:port." -) # type: Any +) # type: Callable[..., Option] retries = partial( Option, @@ -219,7 +240,7 @@ retries = partial( default=5, help="Maximum number of retries each connection should attempt " "(default %default times).", -) # type: Any +) # type: Callable[..., Option] timeout = partial( Option, @@ -229,7 +250,7 @@ timeout = partial( type='float', default=15, help='Set the socket timeout (default %default seconds).', -) # type: Any +) # type: Callable[..., Option] skip_requirements_regex = partial( Option, @@ -239,10 +260,11 @@ skip_requirements_regex = partial( type='str', default='', help=SUPPRESS_HELP, -) # type: Any +) # type: Callable[..., Option] def exists_action(): + # type: () -> Option return Option( # Option when path already exist '--exists-action', @@ -264,7 +286,7 @@ cert = partial( type='str', metavar='path', help="Path to alternate CA bundle.", -) # type: Any +) # type: Callable[..., Option] client_cert = partial( Option, @@ -275,7 +297,7 @@ client_cert = partial( metavar='path', help="Path to SSL client certificate, a single file containing the " "private key and the certificate in PEM format.", -) # type: Any +) # type: Callable[..., Option] index_url = partial( Option, @@ -287,7 +309,7 @@ index_url = partial( "This should point to a repository compliant with PEP 503 " "(the simple repository API) or a local directory laid out " "in the same format.", -) # type: Any +) # type: Callable[..., Option] def extra_index_url(): @@ -310,10 +332,11 @@ no_index = partial( action='store_true', default=False, help='Ignore package index (only looking at --find-links URLs instead).', -) # type: Any +) # type: Callable[..., Option] def find_links(): + # type: () -> Option return Option( '-f', '--find-links', dest='find_links', @@ -327,6 +350,7 @@ def find_links(): def trusted_host(): + # type: () -> Option return Option( "--trusted-host", dest="trusted_hosts", @@ -338,18 +362,8 @@ def trusted_host(): ) -# Remove after 1.5 -process_dependency_links = partial( - Option, - "--process-dependency-links", - dest="process_dependency_links", - action="store_true", - default=False, - help="Enable the processing of dependency links.", -) # type: Any - - def constraints(): + # type: () -> Option return Option( '-c', '--constraint', dest='constraints', @@ -362,6 +376,7 @@ def constraints(): def requirements(): + # type: () -> Option return Option( '-r', '--requirement', dest='requirements', @@ -374,6 +389,7 @@ def requirements(): def editable(): + # type: () -> Option return Option( '-e', '--editable', dest='editables', @@ -394,15 +410,17 @@ src = partial( help='Directory to check out editable projects into. ' 'The default in a virtualenv is "/src". ' 'The default for global installs is "/src".' -) # type: Any +) # type: Callable[..., Option] def _get_format_control(values, option): + # type: (Values, Option) -> Any """Get a format_control object.""" return getattr(values, option.dest) def _handle_no_binary(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, existing.no_binary, existing.only_binary, @@ -410,6 +428,7 @@ def _handle_no_binary(option, opt_str, value, parser): def _handle_only_binary(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, existing.only_binary, existing.no_binary, @@ -417,6 +436,7 @@ def _handle_only_binary(option, opt_str, value, parser): def no_binary(): + # type: () -> Option format_control = FormatControl(set(), set()) return Option( "--no-binary", dest="format_control", action="callback", @@ -432,6 +452,7 @@ def no_binary(): def only_binary(): + # type: () -> Option format_control = FormatControl(set(), set()) return Option( "--only-binary", dest="format_control", action="callback", @@ -454,7 +475,7 @@ platform = partial( default=None, help=("Only use wheels compatible with . " "Defaults to the platform of the running system."), -) +) # type: Callable[..., Option] python_version = partial( @@ -469,7 +490,7 @@ python_version = partial( "version (e.g. '2') can be specified to match all " "minor revs of that major version. A minor version " "(e.g. '34') can also be specified."), -) +) # type: Callable[..., Option] implementation = partial( @@ -483,7 +504,7 @@ implementation = partial( " or 'ip'. If not specified, then the current " "interpreter implementation is used. Use 'py' to force " "implementation-agnostic wheels."), -) +) # type: Callable[..., Option] abi = partial( @@ -498,10 +519,11 @@ abi = partial( "you will need to specify --implementation, " "--platform, and --python-version when using " "this option."), -) +) # type: Callable[..., Option] def prefer_binary(): + # type: () -> Option return Option( "--prefer-binary", dest="prefer_binary", @@ -518,15 +540,44 @@ cache_dir = partial( default=USER_CACHE_DIR, metavar="dir", help="Store the cache data in ." -) +) # type: Callable[..., Option] + + +def no_cache_dir_callback(option, opt, value, parser): + """ + Process a value provided for the --no-cache-dir option. + + This is an optparse.Option callback for the --no-cache-dir option. + """ + # The value argument will be None if --no-cache-dir is passed via the + # command-line, since the option doesn't accept arguments. However, + # the value can be non-None if the option is triggered e.g. by an + # environment variable, like PIP_NO_CACHE_DIR=true. + if value is not None: + # Then parse the string value to get argument error-checking. + try: + strtobool(value) + except ValueError as exc: + raise_option_error(parser, option=option, msg=str(exc)) + + # Originally, setting PIP_NO_CACHE_DIR to a value that strtobool() + # converted to 0 (like "false" or "no") caused cache_dir to be disabled + # rather than enabled (logic would say the latter). Thus, we disable + # the cache directory not just on values that parse to True, but (for + # backwards compatibility reasons) also on values that parse to False. + # In other words, always set it to False if the option is provided in + # some (valid) form. + parser.values.cache_dir = False + no_cache = partial( Option, "--no-cache-dir", dest="cache_dir", - action="store_false", + action="callback", + callback=no_cache_dir_callback, help="Disable the cache.", -) +) # type: Callable[..., Option] no_deps = partial( Option, @@ -535,7 +586,7 @@ no_deps = partial( action='store_true', default=False, help="Don't install package dependencies.", -) # type: Any +) # type: Callable[..., Option] build_dir = partial( Option, @@ -547,7 +598,7 @@ build_dir = partial( 'The location of temporary directories can be controlled by setting ' 'the TMPDIR environment variable (TEMP on Windows) appropriately. ' 'When passed, build directories are not cleaned in case of failures.' -) # type: Any +) # type: Callable[..., Option] ignore_requires_python = partial( Option, @@ -555,7 +606,7 @@ ignore_requires_python = partial( dest='ignore_requires_python', action='store_true', help='Ignore the Requires-Python information.' -) # type: Any +) # type: Callable[..., Option] no_build_isolation = partial( Option, @@ -566,6 +617,50 @@ no_build_isolation = partial( help='Disable isolation when building a modern source distribution. ' 'Build dependencies specified by PEP 518 must be already installed ' 'if this option is used.' +) # type: Callable[..., Option] + + +def no_use_pep517_callback(option, opt, value, parser): + """ + Process a value provided for the --no-use-pep517 option. + + This is an optparse.Option callback for the no_use_pep517 option. + """ + # Since --no-use-pep517 doesn't accept arguments, the value argument + # will be None if --no-use-pep517 is passed via the command-line. + # However, the value can be non-None if the option is triggered e.g. + # by an environment variable, for example "PIP_NO_USE_PEP517=true". + if value is not None: + msg = """A value was passed for --no-use-pep517, + probably using either the PIP_NO_USE_PEP517 environment variable + or the "no-use-pep517" config file option. Use an appropriate value + of the PIP_USE_PEP517 environment variable or the "use-pep517" + config file option instead. + """ + raise_option_error(parser, option=option, msg=msg) + + # Otherwise, --no-use-pep517 was passed via the command-line. + parser.values.use_pep517 = False + + +use_pep517 = partial( + Option, + '--use-pep517', + dest='use_pep517', + action='store_true', + default=None, + help='Use PEP 517 for building source distributions ' + '(use --no-use-pep517 to force legacy behaviour).' +) # type: Any + +no_use_pep517 = partial( + Option, + '--no-use-pep517', + dest='use_pep517', + action='callback', + callback=no_use_pep517_callback, + default=None, + help=SUPPRESS_HELP ) # type: Any install_options = partial( @@ -579,7 +674,7 @@ install_options = partial( "bin\"). Use multiple --install-option options to pass multiple " "options to setup.py install. If you are using an option with a " "directory path, be sure to use absolute path.", -) # type: Any +) # type: Callable[..., Option] global_options = partial( Option, @@ -589,7 +684,7 @@ global_options = partial( metavar='options', help="Extra global options to be supplied to the setup.py " "call before the install command.", -) # type: Any +) # type: Callable[..., Option] no_clean = partial( Option, @@ -597,7 +692,7 @@ no_clean = partial( action='store_true', default=False, help="Don't clean up build directories." -) # type: Any +) # type: Callable[..., Option] pre = partial( Option, @@ -606,7 +701,7 @@ pre = partial( default=False, help="Include pre-release and development versions. By default, " "pip only finds stable versions.", -) # type: Any +) # type: Callable[..., Option] disable_pip_version_check = partial( Option, @@ -616,7 +711,7 @@ disable_pip_version_check = partial( default=False, help="Don't periodically check PyPI to determine whether a new version " "of pip is available for download. Implied with --no-index.", -) # type: Any +) # type: Callable[..., Option] # Deprecated, Remove later @@ -626,14 +721,15 @@ always_unzip = partial( dest='always_unzip', action='store_true', help=SUPPRESS_HELP, -) # type: Any +) # type: Callable[..., Option] def _merge_hash(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None """Given a value spelled "algo:digest", append the digest to a list pointed to in a dict by the algo name.""" if not parser.values.hashes: - parser.values.hashes = {} + parser.values.hashes = {} # type: ignore try: algo, digest = value.split(':', 1) except ValueError: @@ -657,7 +753,7 @@ hash = partial( type='string', help="Verify that the package's archive matches this " 'hash before installing. Example: --hash=sha256:abcdef...', -) # type: Any +) # type: Callable[..., Option] require_hashes = partial( @@ -669,7 +765,7 @@ require_hashes = partial( help='Require a hash to check each requirement against, for ' 'repeatable installs. This option is implied when any package in a ' 'requirements file has a --hash option.', -) # type: Any +) # type: Callable[..., Option] ########## @@ -700,7 +796,7 @@ general_group = { disable_pip_version_check, no_color, ] -} +} # type: Dict[str, Any] index_group = { 'name': 'Package Index Options', @@ -709,6 +805,5 @@ index_group = { extra_index_url, no_index, find_links, - process_dependency_links, ] -} +} # type: Dict[str, Any] diff --git a/pipenv/patched/notpip/_internal/cli/main_parser.py b/pipenv/patched/notpip/_internal/cli/main_parser.py index abe2f69e..704bf404 100644 --- a/pipenv/patched/notpip/_internal/cli/main_parser.py +++ b/pipenv/patched/notpip/_internal/cli/main_parser.py @@ -14,11 +14,17 @@ from pipenv.patched.notpip._internal.commands import ( ) from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.utils.misc import get_prog +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Tuple, List # noqa: F401 + __all__ = ["create_main_parser", "parse_command"] def create_main_parser(): + # type: () -> ConfigOptionParser """Creates and returns the main parser for pip's CLI """ @@ -44,7 +50,8 @@ def create_main_parser(): gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) parser.add_option_group(gen_opts) - parser.main = True # so the help formatter knows + # so the help formatter knows + parser.main = True # type: ignore # create command listing for description command_summaries = get_summaries() @@ -55,6 +62,7 @@ def create_main_parser(): def parse_command(args): + # type: (List[str]) -> Tuple[str, List[str]] parser = create_main_parser() # Note: parser calls disable_interspersed_args(), so the result of this @@ -68,7 +76,7 @@ def parse_command(args): # --version if general_options.version: - sys.stdout.write(parser.version) + sys.stdout.write(parser.version) # type: ignore sys.stdout.write(os.linesep) sys.exit() diff --git a/pipenv/patched/notpip/_internal/commands/check.py b/pipenv/patched/notpip/_internal/commands/check.py index adf4f5e7..cf84f5df 100644 --- a/pipenv/patched/notpip/_internal/commands/check.py +++ b/pipenv/patched/notpip/_internal/commands/check.py @@ -16,7 +16,7 @@ class CheckCommand(Command): summary = 'Verify installed packages have compatible dependencies.' def run(self, options, args): - package_set = create_package_set_from_installed() + package_set, parsing_probs = create_package_set_from_installed() missing, conflicting = check_package_set(package_set) for project_name in missing: @@ -35,7 +35,7 @@ class CheckCommand(Command): project_name, version, req, dep_name, dep_version, ) - if missing or conflicting: + if missing or conflicting or parsing_probs: return 1 else: logger.info("No broken requirements found.") diff --git a/pipenv/patched/notpip/_internal/commands/download.py b/pipenv/patched/notpip/_internal/commands/download.py index e5d87121..133ca135 100644 --- a/pipenv/patched/notpip/_internal/commands/download.py +++ b/pipenv/patched/notpip/_internal/commands/download.py @@ -58,6 +58,8 @@ class DownloadCommand(RequirementCommand): cmd_opts.add_option(cmdoptions.require_hashes()) cmd_opts.add_option(cmdoptions.progress_bar()) cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) cmd_opts.add_option( '-d', '--dest', '--destination-dir', '--destination-directory', diff --git a/pipenv/patched/notpip/_internal/commands/install.py b/pipenv/patched/notpip/_internal/commands/install.py index ddcb4759..68255c85 100644 --- a/pipenv/patched/notpip/_internal/commands/install.py +++ b/pipenv/patched/notpip/_internal/commands/install.py @@ -30,12 +30,6 @@ from pipenv.patched.notpip._internal.utils.misc import ( from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.wheel import WheelBuilder -try: - import wheel -except ImportError: - wheel = None - - logger = logging.getLogger(__name__) @@ -158,6 +152,8 @@ class InstallCommand(RequirementCommand): cmd_opts.add_option(cmdoptions.ignore_requires_python()) cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) cmd_opts.add_option(cmdoptions.install_options()) cmd_opts.add_option(cmdoptions.global_options()) @@ -314,6 +310,7 @@ class InstallCommand(RequirementCommand): ignore_requires_python=options.ignore_requires_python, ignore_installed=options.ignore_installed, isolated=options.isolated_mode, + use_pep517=options.use_pep517 ) resolver.resolve(requirement_set) @@ -321,21 +318,51 @@ class InstallCommand(RequirementCommand): modifying_pip=requirement_set.has_requirement("pip") ) - # If caching is disabled or wheel is not installed don't - # try to build wheels. - if wheel and options.cache_dir: - # build wheels before install. - wb = WheelBuilder( - finder, preparer, wheel_cache, - build_options=[], global_options=[], - ) - # Ignore the result: a failed wheel will be - # installed from the sdist/vcs whatever. + # Consider legacy and PEP517-using requirements separately + legacy_requirements = [] + pep517_requirements = [] + for req in requirement_set.requirements.values(): + if req.use_pep517: + pep517_requirements.append(req) + else: + legacy_requirements.append(req) + + # We don't build wheels for legacy requirements if we + # don't have wheel installed or we don't have a cache dir + try: + import wheel # noqa: F401 + build_legacy = bool(options.cache_dir) + except ImportError: + build_legacy = False + + wb = WheelBuilder( + finder, preparer, wheel_cache, + build_options=[], global_options=[], + ) + + # Always build PEP 517 requirements + build_failures = wb.build( + pep517_requirements, + session=session, autobuilding=True + ) + + if build_legacy: + # We don't care about failures building legacy + # requirements, as we'll fall through to a direct + # install for those. wb.build( - requirement_set.requirements.values(), + legacy_requirements, session=session, autobuilding=True ) + # If we're using PEP 517, we cannot do a direct install + # so we fail here. + if build_failures: + raise InstallationError( + "Could not build wheels for {} which use" + " PEP 517 and cannot be installed directly".format( + ", ".join(r.name for r in build_failures))) + to_install = resolver.get_installation_order( requirement_set ) @@ -472,7 +499,11 @@ class InstallCommand(RequirementCommand): ) def _warn_about_conflicts(self, to_install): - package_set, _dep_info = check_install_conflicts(to_install) + try: + package_set, _dep_info = check_install_conflicts(to_install) + except Exception: + logger.error("Error checking for conflicts.", exc_info=True) + return missing, conflicting = _dep_info # NOTE: There is some duplication here from pipenv.patched.notpip check diff --git a/pipenv/patched/notpip/_internal/commands/list.py b/pipenv/patched/notpip/_internal/commands/list.py index 577c0b5f..a2bd5be1 100644 --- a/pipenv/patched/notpip/_internal/commands/list.py +++ b/pipenv/patched/notpip/_internal/commands/list.py @@ -118,7 +118,6 @@ class ListCommand(Command): index_urls=index_urls, allow_all_prereleases=options.pre, trusted_hosts=options.trusted_hosts, - process_dependency_links=options.process_dependency_links, session=session, ) @@ -134,14 +133,18 @@ class ListCommand(Command): include_editables=options.include_editable, ) + # get_not_required must be called firstly in order to find and + # filter out all dependencies correctly. Otherwise a package + # can't be identified as requirement because some parent packages + # could be filtered out before. + if options.not_required: + packages = self.get_not_required(packages, options) + if options.outdated: packages = self.get_outdated(packages, options) elif options.uptodate: packages = self.get_uptodate(packages, options) - if options.not_required: - packages = self.get_not_required(packages, options) - self.output_package_listing(packages, options) def get_outdated(self, packages, options): @@ -168,16 +171,8 @@ class ListCommand(Command): logger.debug('Ignoring indexes: %s', ','.join(index_urls)) index_urls = [] - dependency_links = [] - for dist in packages: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend( - dist.get_metadata_lines('dependency_links.txt'), - ) - with self._build_session(options) as session: finder = self._build_package_finder(options, index_urls, session) - finder.add_dependency_links(dependency_links) for dist in packages: typ = 'unknown' diff --git a/pipenv/patched/notpip/_internal/commands/wheel.py b/pipenv/patched/notpip/_internal/commands/wheel.py index 08d695ab..801efff8 100644 --- a/pipenv/patched/notpip/_internal/commands/wheel.py +++ b/pipenv/patched/notpip/_internal/commands/wheel.py @@ -67,6 +67,8 @@ class WheelCommand(RequirementCommand): help="Extra arguments to be supplied to 'setup.py bdist_wheel'.", ) cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) cmd_opts.add_option(cmdoptions.constraints()) cmd_opts.add_option(cmdoptions.editable()) cmd_opts.add_option(cmdoptions.requirements()) @@ -157,6 +159,7 @@ class WheelCommand(RequirementCommand): ignore_requires_python=options.ignore_requires_python, ignore_installed=True, isolated=options.isolated_mode, + use_pep517=options.use_pep517 ) resolver.resolve(requirement_set) @@ -167,10 +170,10 @@ class WheelCommand(RequirementCommand): global_options=options.global_options or [], no_clean=options.no_clean, ) - wheels_built_successfully = wb.build( + build_failures = wb.build( requirement_set.requirements.values(), session=session, ) - if not wheels_built_successfully: + if len(build_failures) != 0: raise CommandError( "Failed to build one or more wheels" ) diff --git a/pipenv/patched/notpip/_internal/download.py b/pipenv/patched/notpip/_internal/download.py index 8f4c38f5..f593c2f2 100644 --- a/pipenv/patched/notpip/_internal/download.py +++ b/pipenv/patched/notpip/_internal/download.py @@ -19,7 +19,6 @@ from pipenv.patched.notpip._vendor.lockfile import LockError from pipenv.patched.notpip._vendor.requests.adapters import BaseAdapter, HTTPAdapter from pipenv.patched.notpip._vendor.requests.auth import AuthBase, HTTPBasicAuth from pipenv.patched.notpip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response -from pipenv.patched.notpip._vendor.requests.sessions import Session from pipenv.patched.notpip._vendor.requests.structures import CaseInsensitiveDict from pipenv.patched.notpip._vendor.requests.utils import get_netrc_auth # NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is @@ -27,7 +26,6 @@ from pipenv.patched.notpip._vendor.requests.utils import get_netrc_auth from pipenv.patched.notpip._vendor.six.moves import xmlrpc_client # type: ignore from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._vendor.six.moves.urllib.parse import unquote as urllib_unquote from pipenv.patched.notpip._vendor.urllib3.util import IS_PYOPENSSL import pipenv.patched.notpip @@ -40,14 +38,23 @@ from pipenv.patched.notpip._internal.utils.glibc import libc_ver from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( ARCHIVE_EXTENSIONS, ask_path_exists, backup_dir, call_subprocess, consume, - display_path, format_size, get_installed_version, rmtree, splitext, - unpack_file, + display_path, format_size, get_installed_version, rmtree, + split_auth_from_netloc, splitext, unpack_file, ) from pipenv.patched.notpip._internal.utils.setuptools_build import SETUPTOOLS_SHIM from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import DownloadProgressProvider from pipenv.patched.notpip._internal.vcs import vcs +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Optional, Tuple, Dict, IO, Text, Union + ) + from pipenv.patched.notpip._internal.models.link import Link # noqa: F401 + from pipenv.patched.notpip._internal.utils.hashes import Hashes # noqa: F401 + from pipenv.patched.notpip._internal.vcs import AuthInfo # noqa: F401 + try: import ssl # noqa except ImportError: @@ -137,14 +144,15 @@ def user_agent(): class MultiDomainBasicAuth(AuthBase): def __init__(self, prompting=True): + # type: (bool) -> None self.prompting = prompting - self.passwords = {} + self.passwords = {} # type: Dict[str, AuthInfo] def __call__(self, req): parsed = urllib_parse.urlparse(req.url) - # Get the netloc without any embedded credentials - netloc = parsed.netloc.rsplit("@", 1)[-1] + # Split the credentials from the netloc. + netloc, url_user_password = split_auth_from_netloc(parsed.netloc) # Set the url of the request to the url without any credentials req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:]) @@ -152,9 +160,9 @@ class MultiDomainBasicAuth(AuthBase): # Use any stored credentials that we have for this netloc username, password = self.passwords.get(netloc, (None, None)) - # Extract credentials embedded in the url if we have none stored + # Use the credentials embedded in the url if we have none stored if username is None: - username, password = self.parse_credentials(parsed.netloc) + username, password = url_user_password # Get creds from netrc if we still don't have them if username is None and password is None: @@ -200,6 +208,7 @@ class MultiDomainBasicAuth(AuthBase): # Add our new username and password to the request req = HTTPBasicAuth(username or "", password or "")(resp.request) + req.register_hook("response", self.warn_on_401) # Send our new request new_resp = resp.connection.send(req, **kwargs) @@ -207,14 +216,11 @@ class MultiDomainBasicAuth(AuthBase): return new_resp - def parse_credentials(self, netloc): - if "@" in netloc: - userinfo = netloc.rsplit("@", 1)[0] - if ":" in userinfo: - user, pwd = userinfo.split(":", 1) - return (urllib_unquote(user), urllib_unquote(pwd)) - return urllib_unquote(userinfo), None - return None, None + def warn_on_401(self, resp, **kwargs): + # warn user that they provided incorrect credentials + if resp.status_code == 401: + logger.warning('401 Error, Credentials not correct for %s', + resp.request.url) class LocalFSAdapter(BaseAdapter): @@ -325,7 +331,7 @@ class InsecureHTTPAdapter(HTTPAdapter): class PipSession(requests.Session): - timeout = None + timeout = None # type: Optional[int] def __init__(self, *args, **kwargs): retries = kwargs.pop("retries", 0) @@ -398,6 +404,7 @@ class PipSession(requests.Session): def get_file_content(url, comes_from=None, session=None): + # type: (str, Optional[str], Optional[PipSession]) -> Tuple[str, Text] """Gets the content of a file; it may be a filename, file: URL, or http: URL. Returns (location, content). Content is unicode. @@ -448,6 +455,7 @@ _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) def is_url(name): + # type: (Union[str, Text]) -> bool """Returns true if the name looks like a URL""" if ':' not in name: return False @@ -456,6 +464,7 @@ def is_url(name): def url_to_path(url): + # type: (str) -> str """ Convert a file: URL to a path. """ @@ -473,6 +482,7 @@ def url_to_path(url): def path_to_url(path): + # type: (Union[str, Text]) -> str """ Convert a path to a file: URL. The path will be made absolute and have quoted path parts. @@ -483,6 +493,7 @@ def path_to_url(path): def is_archive_file(name): + # type: (str) -> bool """Return True if `name` is a considered as an archive file.""" ext = splitext(name)[1].lower() if ext in ARCHIVE_EXTENSIONS: @@ -503,14 +514,17 @@ def _get_used_vcs_backend(link): def is_vcs_url(link): + # type: (Link) -> bool return bool(_get_used_vcs_backend(link)) def is_file_url(link): + # type: (Link) -> bool return link.url.lower().startswith('file:') def is_dir_url(link): + # type: (Link) -> bool """Return whether a file:// Link points to a directory. ``link`` must not have any other scheme but file://. Call is_file_url() @@ -525,7 +539,14 @@ def _progress_indicator(iterable, *args, **kwargs): return iterable -def _download_url(resp, link, content_file, hashes, progress_bar): +def _download_url( + resp, # type: Response + link, # type: Link + content_file, # type: IO + hashes, # type: Hashes + progress_bar # type: str +): + # type: (...) -> None try: total_length = int(resp.headers['content-length']) except (ValueError, KeyError, TypeError): @@ -647,8 +668,15 @@ def _copy_file(filename, location, link): logger.info('Saved %s', display_path(download_location)) -def unpack_http_url(link, location, download_dir=None, - session=None, hashes=None, progress_bar="on"): +def unpack_http_url( + link, # type: Link + location, # type: str + download_dir=None, # type: Optional[str] + session=None, # type: Optional[PipSession] + hashes=None, # type: Optional[Hashes] + progress_bar="on" # type: str +): + # type: (...) -> None if session is None: raise TypeError( "unpack_http_url() missing 1 required keyword argument: 'session'" @@ -685,7 +713,13 @@ def unpack_http_url(link, location, download_dir=None, os.unlink(from_path) -def unpack_file_url(link, location, download_dir=None, hashes=None): +def unpack_file_url( + link, # type: Link + location, # type: str + download_dir=None, # type: Optional[str] + hashes=None # type: Optional[Hashes] +): + # type: (...) -> None """Unpack link into location. If download_dir is provided and link points to a file, make a copy @@ -798,9 +832,16 @@ class PipXmlrpcTransport(xmlrpc_client.Transport): raise -def unpack_url(link, location, download_dir=None, - only_download=False, session=None, hashes=None, - progress_bar="on"): +def unpack_url( + link, # type: Optional[Link] + location, # type: Optional[str] + download_dir=None, # type: Optional[str] + only_download=False, # type: bool + session=None, # type: Optional[PipSession] + hashes=None, # type: Optional[Hashes] + progress_bar="on" # type: str +): + # type: (...) -> None """Unpack link. If link is a VCS link: if only_download, export into download_dir and ignore location @@ -840,7 +881,14 @@ def unpack_url(link, location, download_dir=None, write_delete_marker_file(location) -def _download_http_url(link, session, temp_dir, hashes, progress_bar): +def _download_http_url( + link, # type: Link + session, # type: PipSession + temp_dir, # type: str + hashes, # type: Hashes + progress_bar # type: str +): + # type: (...) -> Tuple[str, str] """Download link url into temp_dir using provided session""" target_url = link.url.split('#', 1)[0] try: @@ -900,6 +948,7 @@ def _download_http_url(link, session, temp_dir, hashes, progress_bar): def _check_download_dir(link, download_dir, hashes): + # type: (Link, str, Hashes) -> Optional[str] """ Check download_dir for previously downloaded file with correct hash If a correct file is found return its path else None """ diff --git a/pipenv/patched/notpip/_internal/exceptions.py b/pipenv/patched/notpip/_internal/exceptions.py index 2eadcf28..1342935d 100644 --- a/pipenv/patched/notpip/_internal/exceptions.py +++ b/pipenv/patched/notpip/_internal/exceptions.py @@ -5,6 +5,12 @@ from itertools import chain, groupby, repeat from pipenv.patched.notpip._vendor.six import iteritems +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + class PipError(Exception): """Base pip exception""" @@ -96,7 +102,7 @@ class HashError(InstallationError): typically available earlier. """ - req = None + req = None # type: Optional[InstallRequirement] head = '' def body(self): diff --git a/pipenv/patched/notpip/_internal/index.py b/pipenv/patched/notpip/_internal/index.py index b4b02373..ad145fe4 100644 --- a/pipenv/patched/notpip/_internal/index.py +++ b/pipenv/patched/notpip/_internal/index.py @@ -16,7 +16,7 @@ from pipenv.patched.notpip._vendor.distlib.compat import unescape from pipenv.patched.notpip._vendor.packaging import specifiers from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version -from pipenv.patched.notpip._vendor.requests.exceptions import SSLError +from pipenv.patched.notpip._vendor.requests.exceptions import RetryError, SSLError from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request @@ -31,14 +31,29 @@ from pipenv.patched.notpip._internal.models.index import PyPI from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.pep425tags import get_supported from pipenv.patched.notpip._internal.utils.compat import ipaddress -from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, normalize_path, - remove_auth_from_url, + ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, WHEEL_EXTENSION, normalize_path, + redact_password_from_url, ) from pipenv.patched.notpip._internal.utils.packaging import check_requires_python -from pipenv.patched.notpip._internal.wheel import Wheel, wheel_ext +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.wheel import Wheel + +if MYPY_CHECK_RUNNING: + from logging import Logger # noqa: F401 + from typing import ( # noqa: F401 + Tuple, Optional, Any, List, Union, Callable, Set, Sequence, + Iterable, MutableMapping + ) + from pipenv.patched.notpip._vendor.packaging.version import _BaseVersion # noqa: F401 + from pipenv.patched.notpip._vendor.requests import Response # noqa: F401 + from pipenv.patched.notpip._internal.req import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + + SecureOrigin = Tuple[str, str, Optional[str]] + BuildTag = Tuple[Any, ...] # either emply tuple or Tuple[int, str] + CandidateSortingKey = Tuple[int, _BaseVersion, BuildTag, Optional[int]] __all__ = ['FormatControl', 'PackageFinder'] @@ -53,126 +68,190 @@ SECURE_ORIGINS = [ ("file", "*", None), # ssh is always secure. ("ssh", "*", "*"), -] +] # type: List[SecureOrigin] logger = logging.getLogger(__name__) -def _get_content_type(url, session): - """Get the Content-Type of the given url, using a HEAD request""" +def _match_vcs_scheme(url): + # type: (str) -> Optional[str] + """Look for VCS schemes in the URL. + + Returns the matched VCS scheme, or None if there's no match. + """ + from pipenv.patched.notpip._internal.vcs import VcsSupport + for scheme in VcsSupport.schemes: + if url.lower().startswith(scheme) and url[len(scheme)] in '+:': + return scheme + return None + + +def _is_url_like_archive(url): + # type: (str) -> bool + """Return whether the URL looks like an archive. + """ + filename = Link(url).filename + for bad_ext in ARCHIVE_EXTENSIONS: + if filename.endswith(bad_ext): + return True + return False + + +class _NotHTML(Exception): + def __init__(self, content_type, request_desc): + # type: (str, str) -> None + super(_NotHTML, self).__init__(content_type, request_desc) + self.content_type = content_type + self.request_desc = request_desc + + +def _ensure_html_header(response): + # type: (Response) -> None + """Check the Content-Type header to ensure the response contains HTML. + + Raises `_NotHTML` if the content type is not text/html. + """ + content_type = response.headers.get("Content-Type", "") + if not content_type.lower().startswith("text/html"): + raise _NotHTML(content_type, response.request.method) + + +class _NotHTTP(Exception): + pass + + +def _ensure_html_response(url, session): + # type: (str, PipSession) -> None + """Send a HEAD request to the URL, and ensure the response contains HTML. + + Raises `_NotHTTP` if the URL is not available for a HEAD request, or + `_NotHTML` if the content type is not text/html. + """ scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) if scheme not in {'http', 'https'}: - # FIXME: some warning or something? - # assertion error? - return '' + raise _NotHTTP() resp = session.head(url, allow_redirects=True) resp.raise_for_status() - return resp.headers.get("Content-Type", "") + _ensure_html_header(resp) -def _handle_get_page_fail(link, reason, url, meth=None): +def _get_html_response(url, session): + # type: (str, PipSession) -> Response + """Access an HTML page with GET, and return the response. + + This consists of three parts: + + 1. If the URL looks suspiciously like an archive, send a HEAD first to + check the Content-Type is HTML, to avoid downloading a large file. + Raise `_NotHTTP` if the content type cannot be determined, or + `_NotHTML` if it is not HTML. + 2. Actually perform the request. Raise HTTP exceptions on network failures. + 3. Check the Content-Type header to make sure we got HTML, and raise + `_NotHTML` otherwise. + """ + if _is_url_like_archive(url): + _ensure_html_response(url, session=session) + + logger.debug('Getting page %s', url) + + resp = session.get( + url, + headers={ + "Accept": "text/html", + # We don't want to blindly returned cached data for + # /simple/, because authors generally expecting that + # twine upload && pip install will function, but if + # they've done a pip install in the last ~10 minutes + # it won't. Thus by setting this to zero we will not + # blindly use any cached data, however the benefit of + # using max-age=0 instead of no-cache, is that we will + # still support conditional requests, so we will still + # minimize traffic sent in cases where the page hasn't + # changed at all, we will just always incur the round + # trip for the conditional GET now instead of only + # once per 10 minutes. + # For more information, please see pypa/pip#5670. + "Cache-Control": "max-age=0", + }, + ) + resp.raise_for_status() + + # The check for archives above only works if the url ends with + # something that looks like an archive. However that is not a + # requirement of an url. Unless we issue a HEAD request on every + # url we cannot know ahead of time for sure if something is HTML + # or not. However we can check after we've downloaded it. + _ensure_html_header(resp) + + return resp + + +def _handle_get_page_fail( + link, # type: Link + reason, # type: Union[str, Exception] + meth=None # type: Optional[Callable[..., None]] +): + # type: (...) -> None if meth is None: meth = logger.debug meth("Could not fetch URL %s: %s - skipping", link, reason) def _get_html_page(link, session=None): + # type: (Link, Optional[PipSession]) -> Optional[HTMLPage] if session is None: raise TypeError( "_get_html_page() missing 1 required keyword argument: 'session'" ) - url = link.url - url = url.split('#', 1)[0] + url = link.url.split('#', 1)[0] # Check for VCS schemes that do not support lookup as web pages. - from pipenv.patched.notpip._internal.vcs import VcsSupport - for scheme in VcsSupport.schemes: - if url.lower().startswith(scheme) and url[len(scheme)] in '+:': - logger.debug('Cannot look at %s URL %s', scheme, link) - return None + vcs_scheme = _match_vcs_scheme(url) + if vcs_scheme: + logger.debug('Cannot look at %s URL %s', vcs_scheme, link) + return None + + # Tack index.html onto file:// URLs that point to directories + scheme, _, path, _, _, _ = urllib_parse.urlparse(url) + if (scheme == 'file' and os.path.isdir(urllib_request.url2pathname(path))): + # add trailing slash if not present so urljoin doesn't trim + # final segment + if not url.endswith('/'): + url += '/' + url = urllib_parse.urljoin(url, 'index.html') + logger.debug(' file: URL is directory, getting %s', url) try: - filename = link.filename - for bad_ext in ARCHIVE_EXTENSIONS: - if filename.endswith(bad_ext): - content_type = _get_content_type(url, session=session) - if content_type.lower().startswith('text/html'): - break - else: - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - logger.debug('Getting page %s', url) - - # Tack index.html onto file:// URLs that point to directories - (scheme, netloc, path, params, query, fragment) = \ - urllib_parse.urlparse(url) - if (scheme == 'file' and - os.path.isdir(urllib_request.url2pathname(path))): - # add trailing slash if not present so urljoin doesn't trim - # final segment - if not url.endswith('/'): - url += '/' - url = urllib_parse.urljoin(url, 'index.html') - logger.debug(' file: URL is directory, getting %s', url) - - resp = session.get( - url, - headers={ - "Accept": "text/html", - # We don't want to blindly returned cached data for - # /simple/, because authors generally expecting that - # twine upload && pip install will function, but if - # they've done a pip install in the last ~10 minutes - # it won't. Thus by setting this to zero we will not - # blindly use any cached data, however the benefit of - # using max-age=0 instead of no-cache, is that we will - # still support conditional requests, so we will still - # minimize traffic sent in cases where the page hasn't - # changed at all, we will just always incur the round - # trip for the conditional GET now instead of only - # once per 10 minutes. - # For more information, please see pypa/pip#5670. - "Cache-Control": "max-age=0", - }, + resp = _get_html_response(url, session=session) + except _NotHTTP as exc: + logger.debug( + 'Skipping page %s because it looks like an archive, and cannot ' + 'be checked by HEAD.', link, + ) + except _NotHTML as exc: + logger.debug( + 'Skipping page %s because the %s request got Content-Type: %s', + link, exc.request_desc, exc.content_type, ) - resp.raise_for_status() - - # The check for archives above only works if the url ends with - # something that looks like an archive. However that is not a - # requirement of an url. Unless we issue a HEAD request on every - # url we cannot know ahead of time for sure if something is HTML - # or not. However we can check after we've downloaded it. - content_type = resp.headers.get('Content-Type', 'unknown') - if not content_type.lower().startswith("text/html"): - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - inst = HTMLPage(resp.content, resp.url, resp.headers) except requests.HTTPError as exc: - _handle_get_page_fail(link, exc, url) + _handle_get_page_fail(link, exc) + except RetryError as exc: + _handle_get_page_fail(link, exc) except SSLError as exc: reason = "There was a problem confirming the ssl certificate: " reason += str(exc) - _handle_get_page_fail(link, reason, url, meth=logger.info) + _handle_get_page_fail(link, reason, meth=logger.info) except requests.ConnectionError as exc: - _handle_get_page_fail(link, "connection error: %s" % exc, url) + _handle_get_page_fail(link, "connection error: %s" % exc) except requests.Timeout: - _handle_get_page_fail(link, "timed out", url) + _handle_get_page_fail(link, "timed out") else: - return inst + return HTMLPage(resp.content, resp.url, resp.headers) + return None class PackageFinder(object): @@ -182,11 +261,21 @@ class PackageFinder(object): packages, by reading pages and looking for appropriate links. """ - def __init__(self, find_links, index_urls, allow_all_prereleases=False, - trusted_hosts=None, process_dependency_links=False, - session=None, format_control=None, platform=None, - versions=None, abi=None, implementation=None, - prefer_binary=False): + def __init__( + self, + find_links, # type: List[str] + index_urls, # type: List[str] + allow_all_prereleases=False, # type: bool + trusted_hosts=None, # type: Optional[Iterable[str]] + session=None, # type: Optional[PipSession] + format_control=None, # type: Optional[FormatControl] + platform=None, # type: Optional[str] + versions=None, # type: Optional[List[str]] + abi=None, # type: Optional[str] + implementation=None, # type: Optional[str] + prefer_binary=False # type: bool + ): + # type: (...) -> None """Create a PackageFinder. :param format_control: A FormatControl object or None. Used to control @@ -215,7 +304,7 @@ class PackageFinder(object): # it and if it exists, use the normalized version. # This is deliberately conservative - it might be fine just to # blindly normalize anything starting with a ~... - self.find_links = [] + self.find_links = [] # type: List[str] for link in find_links: if link.startswith('~'): new_link = normalize_path(link) @@ -224,10 +313,9 @@ class PackageFinder(object): self.find_links.append(link) self.index_urls = index_urls - self.dependency_links = [] # These are boring links that have already been logged somehow: - self.logged_links = set() + self.logged_links = set() # type: Set[Link] self.format_control = format_control or FormatControl(set(), set()) @@ -235,14 +323,11 @@ class PackageFinder(object): self.secure_origins = [ ("*", host, "*") for host in (trusted_hosts if trusted_hosts else []) - ] + ] # type: List[SecureOrigin] # Do we want to allow _all_ pre-releases? self.allow_all_prereleases = allow_all_prereleases - # Do we process dependency links? - self.process_dependency_links = process_dependency_links - # The Session we'll use to make requests self.session = session @@ -274,11 +359,12 @@ class PackageFinder(object): break def get_formatted_locations(self): + # type: () -> str lines = [] if self.index_urls and self.index_urls != [PyPI.simple_url]: lines.append( "Looking in indexes: {}".format(", ".join( - remove_auth_from_url(url) for url in self.index_urls)) + redact_password_from_url(url) for url in self.index_urls)) ) if self.find_links: lines.append( @@ -286,21 +372,6 @@ class PackageFinder(object): ) return "\n".join(lines) - def add_dependency_links(self, links): - # FIXME: this shouldn't be global list this, it should only - # apply to requirements of the package that specifies the - # dependency_links value - # FIXME: also, we should track comes_from (i.e., use Link) - if self.process_dependency_links: - deprecated( - "Dependency Links processing has been deprecated and will be " - "removed in a future release.", - replacement="PEP 508 URL dependencies", - gone_in="18.2", - issue=4187, - ) - self.dependency_links.extend(links) - @staticmethod def get_extras_links(links): requires = [] @@ -316,12 +387,11 @@ class PackageFinder(object): extras[link[1:-1]] = current_list else: current_list.append(link) - return extras - @staticmethod def _sort_locations(locations, expand_dir=False): + # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] """ Sort locations into "files" (archives) and "urls", and return a pair of lists (files,urls) @@ -354,6 +424,11 @@ class PackageFinder(object): sort_path(os.path.join(path, item)) elif is_file_url: urls.append(url) + else: + logger.warning( + "Path '{0}' is ignored: " + "it is a directory.".format(path), + ) elif os.path.isfile(path): sort_path(path) else: @@ -373,6 +448,7 @@ class PackageFinder(object): return files, urls def _candidate_sort_key(self, candidate, ignore_compatibility=True): + # type: (InstallationCandidate, bool) -> CandidateSortingKey """ Function used to generate link sort key for link tuples. The greater the return value, the more preferred it is. @@ -387,7 +463,7 @@ class PackageFinder(object): with the same version, would have to be considered equal """ support_num = len(self.valid_tags) - build_tag = tuple() + build_tag = tuple() # type: BuildTag binary_preference = 0 if candidate.location.is_wheel: # can raise InvalidWheelFilename @@ -404,7 +480,6 @@ class PackageFinder(object): pri = -(wheel.support_index_min(tags=tags)) except TypeError: pri = -(support_num) - if wheel.build_tag is not None: match = re.match(r'^(\d+)(.*)$', wheel.build_tag) build_tag_groups = match.groups() @@ -414,6 +489,7 @@ class PackageFinder(object): return (binary_preference, candidate.version, build_tag, pri) def _validate_secure_origin(self, logger, location): + # type: (Logger, Link) -> bool # Determine if this url used a secure transport mechanism parsed = urllib_parse.urlparse(str(location)) origin = (parsed.scheme, parsed.hostname, parsed.port) @@ -445,7 +521,9 @@ class PackageFinder(object): network = ipaddress.ip_network( secure_origin[1] if isinstance(secure_origin[1], six.text_type) - else secure_origin[1].decode("utf8") + # setting secure_origin[1] to proper Union[bytes, str] + # creates problems in other places + else secure_origin[1].decode("utf8") # type: ignore ) except ValueError: # We don't have both a valid address or a valid network, so @@ -485,6 +563,7 @@ class PackageFinder(object): return False def _get_index_urls_locations(self, project_name): + # type: (str) -> List[str] """Returns the locations found via self.index_urls Checks the url_name on the main (first in the list) index and @@ -507,9 +586,10 @@ class PackageFinder(object): return [mkurl_pypi_url(url) for url in self.index_urls] def find_all_candidates(self, project_name): + # type: (str) -> List[Optional[InstallationCandidate]] """Find all available InstallationCandidate for project_name - This checks index_urls, find_links and dependency_links. + This checks index_urls and find_links. All versions found are returned as an InstallationCandidate list. See _link_package_versions for details on which files are accepted @@ -519,21 +599,18 @@ class PackageFinder(object): fl_file_loc, fl_url_loc = self._sort_locations( self.find_links, expand_dir=True, ) - dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links) file_locations = (Link(url) for url in itertools.chain( - index_file_loc, fl_file_loc, dep_file_loc, + index_file_loc, fl_file_loc, )) # We trust every url that the user has given us whether it was given - # via --index-url or --find-links - # We explicitly do not trust links that came from dependency_links + # via --index-url or --find-links. # We want to filter out any thing which does not have a secure origin. url_locations = [ link for link in itertools.chain( (Link(url) for url in index_url_loc), (Link(url) for url in fl_url_loc), - (Link(url) for url in dep_url_loc), ) if self._validate_secure_origin(logger, link) ] @@ -564,17 +641,6 @@ class PackageFinder(object): self._package_versions(page.iter_links(), search) ) - dependency_versions = self._package_versions( - (Link(url) for url in self.dependency_links), search - ) - if dependency_versions: - logger.debug( - 'dependency_links found: %s', - ', '.join([ - version.location.url for version in dependency_versions - ]) - ) - file_versions = self._package_versions(file_locations, search) if file_versions: file_versions.sort(reverse=True) @@ -587,12 +653,10 @@ class PackageFinder(object): ) # This is an intentional priority ordering - return ( - file_versions + find_links_versions + page_versions + - dependency_versions - ) + return file_versions + find_links_versions + page_versions def find_requirement(self, req, upgrade, ignore_compatibility=False): + # type: (InstallRequirement, bool, bool) -> Optional[Link] """Try to find a Link matching req Expects req, an InstallRequirement and upgrade, a boolean @@ -692,20 +756,18 @@ class PackageFinder(object): return best_candidate.location def _get_pages(self, locations, project_name): + # type: (Iterable[Link], str) -> Iterable[HTMLPage] """ Yields (page, page_url) from the given locations, skipping locations that have errors. """ - seen = set() + seen = set() # type: Set[Link] for location in locations: if location in seen: continue seen.add(location) - try: - page = self._get_page(location) - except requests.HTTPError: - continue + page = _get_html_page(location, session=self.session) if page is None: continue @@ -714,12 +776,13 @@ class PackageFinder(object): _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$') def _sort_links(self, links): + # type: (Iterable[Link]) -> List[Link] """ Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates """ eggs, no_eggs = [], [] - seen = set() + seen = set() # type: Set[Link] for link in links: if link not in seen: seen.add(link) @@ -729,7 +792,12 @@ class PackageFinder(object): no_eggs.append(link) return no_eggs + eggs - def _package_versions(self, links, search): + def _package_versions( + self, + links, # type: Iterable[Link] + search # type: Search + ): + # type: (...) -> List[Optional[InstallationCandidate]] result = [] for link in self._sort_links(links): v = self._link_package_versions(link, search) @@ -738,11 +806,13 @@ class PackageFinder(object): return result def _log_skipped_link(self, link, reason): + # type: (Link, str) -> None if link not in self.logged_links: logger.debug('Skipping link %s; %s', link, reason) self.logged_links.add(link) def _link_package_versions(self, link, search, ignore_compatibility=True): + # type: (Link, Search, bool) -> Optional[InstallationCandidate] """Return an InstallationCandidate or None""" version = None if link.egg_fragment: @@ -752,51 +822,51 @@ class PackageFinder(object): egg_info, ext = link.splitext() if not ext: self._log_skipped_link(link, 'not a file') - return + return None if ext not in SUPPORTED_EXTENSIONS: self._log_skipped_link( link, 'unsupported archive format: %s' % ext, ) - return - if "binary" not in search.formats and ext == wheel_ext and not ignore_compatibility: + return None + if "binary" not in search.formats and ext == WHEEL_EXTENSION and not ignore_compatibility: self._log_skipped_link( link, 'No binaries permitted for %s' % search.supplied, ) - return + return None if "macosx10" in link.path and ext == '.zip' and not ignore_compatibility: self._log_skipped_link(link, 'macosx10 one') - return - if ext == wheel_ext: + return None + if ext == WHEEL_EXTENSION: try: wheel = Wheel(link.filename) except InvalidWheelFilename: self._log_skipped_link(link, 'invalid wheel filename') - return + return None if canonicalize_name(wheel.name) != search.canonical: self._log_skipped_link( link, 'wrong project name (not %s)' % search.supplied) - return + return None if not wheel.supported(self.valid_tags) and not ignore_compatibility: self._log_skipped_link( link, 'it is not compatible with this Python') - return + return None version = wheel.version # This should be up by the search.ok_binary check, but see issue 2700. - if "source" not in search.formats and ext != wheel_ext: + if "source" not in search.formats and ext != WHEEL_EXTENSION: self._log_skipped_link( link, 'No sources permitted for %s' % search.supplied, ) - return + return None if not version: - version = egg_info_matches(egg_info, search.supplied, link) - if version is None: + version = _egg_info_matches(egg_info, search.canonical) + if not version: self._log_skipped_link( link, 'Missing project version for %s' % search.supplied) - return + return None match = self._py_version_re.search(version) if match: @@ -805,7 +875,7 @@ class PackageFinder(object): if py_version != sys.version[:3]: self._log_skipped_link( link, 'Python version is incorrect') - return + return None try: support_this_python = check_requires_python(link.requires_python) except specifiers.InvalidSpecifier: @@ -814,45 +884,57 @@ class PackageFinder(object): support_this_python = True if not support_this_python and not ignore_compatibility: - logger.debug("The package %s is incompatible with the python" - "version in use. Acceptable python versions are:%s", + logger.debug("The package %s is incompatible with the python " + "version in use. Acceptable python versions are: %s", link, link.requires_python) - return + return None logger.debug('Found link %s, version: %s', link, version) return InstallationCandidate(search.supplied, version, link, link.requires_python) - def _get_page(self, link): - return _get_html_page(link, session=self.session) + +def _find_name_version_sep(egg_info, canonical_name): + # type: (str, str) -> int + """Find the separator's index based on the package's canonical name. + + `egg_info` must be an egg info string for the given package, and + `canonical_name` must be the package's canonical name. + + This function is needed since the canonicalized name does not necessarily + have the same length as the egg info's name part. An example:: + + >>> egg_info = 'foo__bar-1.0' + >>> canonical_name = 'foo-bar' + >>> _find_name_version_sep(egg_info, canonical_name) + 8 + """ + # Project name and version must be separated by one single dash. Find all + # occurrences of dashes; if the string in front of it matches the canonical + # name, this is the one separating the name and version parts. + for i, c in enumerate(egg_info): + if c != "-": + continue + if canonicalize_name(egg_info[:i]) == canonical_name: + return i + raise ValueError("{} does not match {}".format(egg_info, canonical_name)) -def egg_info_matches( - egg_info, search_name, link, - _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): +def _egg_info_matches(egg_info, canonical_name): + # type: (str, str) -> Optional[str] """Pull the version part out of a string. :param egg_info: The string to parse. E.g. foo-2.1 - :param search_name: The name of the package this belongs to. None to - infer the name. Note that this cannot unambiguously parse strings - like foo-2-2 which might be foo, 2-2 or foo-2, 2. - :param link: The link the string came from, for logging on failure. + :param canonical_name: The canonicalized name of the package this + belongs to. """ - match = _egg_info_re.search(egg_info) - if not match: - logger.debug('Could not parse version from link: %s', link) + try: + version_start = _find_name_version_sep(egg_info, canonical_name) + 1 + except ValueError: return None - if search_name is None: - full_match = match.group(0) - return full_match.split('-', 1)[-1] - name = match.group(0).lower() - # To match the "safe" name that pkg_resources creates: - name = name.replace('_', '-') - # project name and version must be separated by a dash - look_for = search_name.lower() + "-" - if name.startswith(look_for): - return match.group(0)[len(look_for):] - else: + version = egg_info[version_start:] + if not version: return None + return version def _determine_base_url(document, page_url): @@ -888,6 +970,7 @@ _CLEAN_LINK_RE = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) def _clean_link(url): + # type: (str) -> str """Makes sure a link is fully encoded. That is, if a ' ' shows up in the link, it will be rewritten to %20 (while not over-quoting % or other characters).""" @@ -898,14 +981,16 @@ class HTMLPage(object): """Represents one page, along with its URL""" def __init__(self, content, url, headers=None): + # type: (bytes, str, MutableMapping[str, str]) -> None self.content = content self.url = url self.headers = headers def __str__(self): - return self.url + return redact_password_from_url(self.url) def iter_links(self): + # type: () -> Iterable[Link] """Yields all links in the page""" document = html5lib.parse( self.content, diff --git a/pipenv/patched/notpip/_internal/locations.py b/pipenv/patched/notpip/_internal/locations.py index 3c7d5bd8..e8f5a268 100644 --- a/pipenv/patched/notpip/_internal/locations.py +++ b/pipenv/patched/notpip/_internal/locations.py @@ -12,6 +12,11 @@ from distutils.command.install import SCHEME_KEYS # type: ignore from pipenv.patched.notpip._internal.utils import appdirs from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any, Union, Dict, List, Optional # noqa: F401 + # Application Directories USER_CACHE_DIR = appdirs.user_cache_dir("pip") @@ -28,6 +33,7 @@ PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' def write_delete_marker_file(directory): + # type: (str) -> None """ Write the pip delete marker file into this directory. """ @@ -37,6 +43,7 @@ def write_delete_marker_file(directory): def running_under_virtualenv(): + # type: () -> bool """ Return True if we're running inside a virtualenv, False otherwise. @@ -50,6 +57,7 @@ def running_under_virtualenv(): def virtualenv_no_global(): + # type: () -> bool """ Return True if in a venv and no system site packages. """ @@ -59,6 +67,8 @@ def virtualenv_no_global(): no_global_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt') if running_under_virtualenv() and os.path.isfile(no_global_file): return True + else: + return False if running_under_virtualenv(): @@ -80,7 +90,8 @@ src_prefix = os.path.abspath(src_prefix) # FIXME doesn't account for venv linked to global site-packages -site_packages = sysconfig.get_path("purelib") +site_packages = sysconfig.get_path("purelib") # type: Optional[str] + # This is because of a bug in PyPy's sysconfig module, see # https://bitbucket.org/pypy/pypy/issues/2506/sysconfig-returns-incorrect-paths # for more information. @@ -135,6 +146,7 @@ new_config_file = os.path.join(appdirs.user_config_dir("pip"), config_basename) def distutils_scheme(dist_name, user=False, home=None, root=None, isolated=False, prefix=None): + # type:(str, bool, str, str, bool, str) -> dict """ Return a distutils install scheme """ @@ -146,12 +158,15 @@ def distutils_scheme(dist_name, user=False, home=None, root=None, extra_dist_args = {"script_args": ["--no-user-cfg"]} else: extra_dist_args = {} - dist_args = {'name': dist_name} + dist_args = {'name': dist_name} # type: Dict[str, Union[str, List[str]]] dist_args.update(extra_dist_args) d = Distribution(dist_args) + # Ignoring, typeshed issue reported python/typeshed/issues/2567 d.parse_config_files() - i = d.get_command_obj('install', create=True) + # NOTE: Ignoring type since mypy can't find attributes on 'Command' + i = d.get_command_obj('install', create=True) # type: Any + assert i is not None # NOTE: setting user or home has the side-effect of creating the home dir # or user base for installations during finalize_options() # ideally, we'd prefer a scheme class that has no side-effects. @@ -171,7 +186,9 @@ def distutils_scheme(dist_name, user=False, home=None, root=None, # platlib). Note, i.install_lib is *always* set after # finalize_options(); we only want to override here if the user # has explicitly requested it hence going back to the config - if 'install_lib' in d.get_option_dict('install'): + + # Ignoring, typeshed issue reported python/typeshed/issues/2567 + if 'install_lib' in d.get_option_dict('install'): # type: ignore scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib)) if running_under_virtualenv(): diff --git a/pipenv/patched/notpip/_internal/models/candidate.py b/pipenv/patched/notpip/_internal/models/candidate.py index 9627589e..adc3550e 100644 --- a/pipenv/patched/notpip/_internal/models/candidate.py +++ b/pipenv/patched/notpip/_internal/models/candidate.py @@ -1,6 +1,12 @@ from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from pipenv.patched.notpip._vendor.packaging.version import _BaseVersion # noqa: F401 + from pipenv.patched.notpip._internal.models.link import Link # noqa: F401 + from typing import Any, Union # noqa: F401 class InstallationCandidate(KeyBasedCompareMixin): @@ -8,8 +14,9 @@ class InstallationCandidate(KeyBasedCompareMixin): """ def __init__(self, project, version, location, requires_python=None): + # type: (Any, str, Link, Any) -> None self.project = project - self.version = parse_version(version) + self.version = parse_version(version) # type: _BaseVersion self.location = location self.requires_python = requires_python @@ -19,6 +26,7 @@ class InstallationCandidate(KeyBasedCompareMixin): ) def __repr__(self): + # type: () -> str return "".format( self.project, self.version, self.location, ) diff --git a/pipenv/patched/notpip/_internal/models/format_control.py b/pipenv/patched/notpip/_internal/models/format_control.py index caad3cba..7172ad9f 100644 --- a/pipenv/patched/notpip/_internal/models/format_control.py +++ b/pipenv/patched/notpip/_internal/models/format_control.py @@ -1,16 +1,24 @@ from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Set, FrozenSet # noqa: F401 + class FormatControl(object): - """A helper class for controlling formats from which packages are installed. - If a field is falsy, it isn't set. If it is {':all:'}, it should match all - packages except those listed in the other field. Only one field can be set - to {':all:'} at a time. The rest of the time exact package name matches - are listed, with any given package only showing up in one field at a time. + """Helper for managing formats from which a package can be installed. """ + def __init__(self, no_binary=None, only_binary=None): - self.no_binary = set() if no_binary is None else no_binary - self.only_binary = set() if only_binary is None else only_binary + # type: (Optional[Set], Optional[Set]) -> None + if no_binary is None: + no_binary = set() + if only_binary is None: + only_binary = set() + + self.no_binary = no_binary + self.only_binary = only_binary def __eq__(self, other): return self.__dict__ == other.__dict__ @@ -27,6 +35,7 @@ class FormatControl(object): @staticmethod def handle_mutual_excludes(value, target, other): + # type: (str, Optional[Set], Optional[Set]) -> None new = value.split(',') while ':all:' in new: other.clear() @@ -45,6 +54,7 @@ class FormatControl(object): target.add(name) def get_allowed_formats(self, canonical_name): + # type: (str) -> FrozenSet result = {"binary", "source"} if canonical_name in self.only_binary: result.discard('source') @@ -57,6 +67,7 @@ class FormatControl(object): return frozenset(result) def disallow_binaries(self): + # type: () -> None self.handle_mutual_excludes( ':all:', self.no_binary, self.only_binary, ) diff --git a/pipenv/patched/notpip/_internal/models/index.py b/pipenv/patched/notpip/_internal/models/index.py index 0983fc9c..b2895dab 100644 --- a/pipenv/patched/notpip/_internal/models/index.py +++ b/pipenv/patched/notpip/_internal/models/index.py @@ -6,6 +6,7 @@ class PackageIndex(object): """ def __init__(self, url, file_storage_domain): + # type: (str, str) -> None super(PackageIndex, self).__init__() self.url = url self.netloc = urllib_parse.urlsplit(url).netloc @@ -18,6 +19,7 @@ class PackageIndex(object): self.file_storage_domain = file_storage_domain def _url_for_path(self, path): + # type: (str) -> str return urllib_parse.urljoin(self.url, path) diff --git a/pipenv/patched/notpip/_internal/models/link.py b/pipenv/patched/notpip/_internal/models/link.py index 686af1d0..ded4de43 100644 --- a/pipenv/patched/notpip/_internal/models/link.py +++ b/pipenv/patched/notpip/_internal/models/link.py @@ -4,9 +4,15 @@ import re from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import splitext +from pipenv.patched.notpip._internal.utils.misc import ( + WHEEL_EXTENSION, redact_password_from_url, splitext, +) from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin -from pipenv.patched.notpip._internal.wheel import wheel_ext +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple, Union, Text # noqa: F401 + from pipenv.patched.notpip._internal.index import HTMLPage # noqa: F401 class Link(KeyBasedCompareMixin): @@ -14,6 +20,7 @@ class Link(KeyBasedCompareMixin): """ def __init__(self, url, comes_from=None, requires_python=None): + # type: (str, Optional[Union[str, HTMLPage]], Optional[str]) -> None """ url: url of the resource pointed to (href of the link) @@ -44,15 +51,17 @@ class Link(KeyBasedCompareMixin): else: rp = '' if self.comes_from: - return '%s (from %s)%s' % (self.url, self.comes_from, rp) + return '%s (from %s)%s' % (redact_password_from_url(self.url), + self.comes_from, rp) else: - return str(self.url) + return redact_password_from_url(str(self.url)) def __repr__(self): return '' % self @property def filename(self): + # type: () -> str _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) name = posixpath.basename(path.rstrip('/')) or netloc name = urllib_parse.unquote(name) @@ -61,25 +70,31 @@ class Link(KeyBasedCompareMixin): @property def scheme(self): + # type: () -> str return urllib_parse.urlsplit(self.url)[0] @property def netloc(self): + # type: () -> str return urllib_parse.urlsplit(self.url)[1] @property def path(self): + # type: () -> str return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) def splitext(self): + # type: () -> Tuple[str, str] return splitext(posixpath.basename(self.path.rstrip('/'))) @property def ext(self): + # type: () -> str return self.splitext()[1] @property def url_without_fragment(self): + # type: () -> str scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) @@ -87,6 +102,7 @@ class Link(KeyBasedCompareMixin): @property def egg_fragment(self): + # type: () -> Optional[str] match = self._egg_fragment_re.search(self.url) if not match: return None @@ -96,6 +112,7 @@ class Link(KeyBasedCompareMixin): @property def subdirectory_fragment(self): + # type: () -> Optional[str] match = self._subdirectory_fragment_re.search(self.url) if not match: return None @@ -107,6 +124,7 @@ class Link(KeyBasedCompareMixin): @property def hash(self): + # type: () -> Optional[str] match = self._hash_re.search(self.url) if match: return match.group(2) @@ -114,6 +132,7 @@ class Link(KeyBasedCompareMixin): @property def hash_name(self): + # type: () -> Optional[str] match = self._hash_re.search(self.url) if match: return match.group(1) @@ -121,14 +140,17 @@ class Link(KeyBasedCompareMixin): @property def show_url(self): + # type: () -> Optional[str] return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) @property def is_wheel(self): - return self.ext == wheel_ext + # type: () -> bool + return self.ext == WHEEL_EXTENSION @property def is_artifact(self): + # type: () -> bool """ Determines if this points to an actual artifact (e.g. a tarball) or if it points to an "abstract" thing like a path or a VCS location. diff --git a/pipenv/patched/notpip/_internal/operations/check.py b/pipenv/patched/notpip/_internal/operations/check.py index 9c8ea08e..a73611d4 100644 --- a/pipenv/patched/notpip/_internal/operations/check.py +++ b/pipenv/patched/notpip/_internal/operations/check.py @@ -1,18 +1,22 @@ """Validation of dependencies of packages """ +import logging from collections import namedtuple from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name +from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError from pipenv.patched.notpip._internal.operations.prepare import make_abstract_dist from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +logger = logging.getLogger(__name__) + if MYPY_CHECK_RUNNING: from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 from typing import ( # noqa: F401 - Any, Callable, Dict, Iterator, Optional, Set, Tuple, List + Any, Callable, Dict, Optional, Set, Tuple, List ) # Shorthands @@ -28,7 +32,7 @@ PackageDetails = namedtuple('PackageDetails', ['version', 'requires']) def create_package_set_from_installed(**kwargs): - # type: (**Any) -> PackageSet + # type: (**Any) -> Tuple[PackageSet, bool] """Converts a list of distributions into a PackageSet. """ # Default to using all packages installed on the system @@ -36,10 +40,16 @@ def create_package_set_from_installed(**kwargs): kwargs = {"local_only": False, "skip": ()} package_set = {} + problems = False for dist in get_installed_distributions(**kwargs): name = canonicalize_name(dist.project_name) - package_set[name] = PackageDetails(dist.version, dist.requires()) - return package_set + try: + package_set[name] = PackageDetails(dist.version, dist.requires()) + except RequirementParseError as e: + # Don't crash on broken metadata + logging.warning("Error parsing requirements for %s: %s", name, e) + problems = True + return package_set, problems def check_package_set(package_set, should_ignore=None): @@ -95,7 +105,7 @@ def check_install_conflicts(to_install): installing given requirements """ # Start from the current state - package_set = create_package_set_from_installed() + package_set, _ = create_package_set_from_installed() # Install packages would_be_installed = _simulate_installation_of(to_install, package_set) @@ -110,9 +120,6 @@ def check_install_conflicts(to_install): ) -# NOTE from @pradyunsg -# This required a minor update in dependency link handling logic over at -# operations.prepare.IsSDist.dist() to get it working def _simulate_installation_of(to_install, package_set): # type: (List[InstallRequirement], PackageSet) -> Set[str] """Computes the version of packages after installing to_install. @@ -123,7 +130,7 @@ def _simulate_installation_of(to_install, package_set): # Modify it as installing requirement_set would (assuming no errors) for inst_req in to_install: - dist = make_abstract_dist(inst_req).dist(finder=None) + dist = make_abstract_dist(inst_req).dist() name = canonicalize_name(dist.key) package_set[name] = PackageDetails(dist.version, dist.requires()) diff --git a/pipenv/patched/notpip/_internal/operations/freeze.py b/pipenv/patched/notpip/_internal/operations/freeze.py index b18b98e4..8fd755e8 100644 --- a/pipenv/patched/notpip/_internal/operations/freeze.py +++ b/pipenv/patched/notpip/_internal/operations/freeze.py @@ -5,57 +5,61 @@ import logging import os import re -from pipenv.patched.notpip._vendor import pkg_resources, six +from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError -from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.exceptions import BadCommand, InstallationError from pipenv.patched.notpip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) from pipenv.patched.notpip._internal.req.req_file import COMMENT_RE -from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.misc import ( - dist_is_editable, get_installed_distributions, make_vcs_requirement_url, + dist_is_editable, get_installed_distributions, ) +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Iterator, Optional, List, Container, Set, Dict, Tuple, Iterable, Union + ) + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + from pipenv.patched.notpip._vendor.pkg_resources import ( # noqa: F401 + Distribution, Requirement + ) + + RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] + logger = logging.getLogger(__name__) def freeze( - requirement=None, - find_links=None, local_only=None, user_only=None, skip_regex=None, - isolated=False, - wheel_cache=None, - exclude_editable=False, - skip=()): + requirement=None, # type: Optional[List[str]] + find_links=None, # type: Optional[List[str]] + local_only=None, # type: Optional[bool] + user_only=None, # type: Optional[bool] + skip_regex=None, # type: Optional[str] + isolated=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + exclude_editable=False, # type: bool + skip=() # type: Container[str] +): + # type: (...) -> Iterator[str] find_links = find_links or [] skip_match = None if skip_regex: skip_match = re.compile(skip_regex).search - dependency_links = [] - - for dist in pkg_resources.working_set: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend( - dist.get_metadata_lines('dependency_links.txt') - ) - for link in find_links: - if '#egg=' in link: - dependency_links.append(link) for link in find_links: yield '-f %s' % link - installations = {} + installations = {} # type: Dict[str, FrozenRequirement] for dist in get_installed_distributions(local_only=local_only, skip=(), user_only=user_only): try: - req = FrozenRequirement.from_dist( - dist, - dependency_links - ) + req = FrozenRequirement.from_dist(dist) except RequirementParseError: logger.warning( "Could not parse requirement: %s", @@ -71,10 +75,10 @@ def freeze( # should only be emitted once, even if the same option is in multiple # requirements files, so we need to keep track of what has been emitted # so that we don't emit it again if it's seen again - emitted_options = set() + emitted_options = set() # type: Set[str] # keep track of which files a requirement is in so that we can # give an accurate warning if a requirement appears multiple times. - req_files = collections.defaultdict(list) + req_files = collections.defaultdict(list) # type: Dict[str, List[str]] for req_file_path in requirement: with open(req_file_path) as req_file: for line in req_file: @@ -128,10 +132,10 @@ def freeze( # but has been processed already if not req_files[line_req.name]: logger.warning( - "Requirement file [%s] contains %s, but that " - "package is not installed", + "Requirement file [%s] contains %s, but " + "package %r is not installed", req_file_path, - COMMENT_RE.sub('', line).strip(), + COMMENT_RE.sub('', line).strip(), line_req.name ) else: req_files[line_req.name].append(req_file_path) @@ -157,105 +161,84 @@ def freeze( yield str(installation).rstrip() +def get_requirement_info(dist): + # type: (Distribution) -> RequirementInfo + """ + Compute and return values (req, editable, comments) for use in + FrozenRequirement.from_dist(). + """ + if not dist_is_editable(dist): + return (None, False, []) + + location = os.path.normcase(os.path.abspath(dist.location)) + + from pipenv.patched.notpip._internal.vcs import vcs, RemoteNotFoundError + vc_type = vcs.get_backend_type(location) + + if not vc_type: + req = dist.as_requirement() + logger.debug( + 'No VCS found for editable requirement {!r} in: {!r}', req, + location, + ) + comments = [ + '# Editable install with no version control ({})'.format(req) + ] + return (location, True, comments) + + try: + req = vc_type.get_src_requirement(location, dist.project_name) + except RemoteNotFoundError: + req = dist.as_requirement() + comments = [ + '# Editable {} install with no remote ({})'.format( + vc_type.__name__, req, + ) + ] + return (location, True, comments) + + except BadCommand: + logger.warning( + 'cannot determine version of editable source in %s ' + '(%s command not found in path)', + location, + vc_type.name, + ) + return (None, True, []) + + except InstallationError as exc: + logger.warning( + "Error when trying to get requirement for VCS system %s, " + "falling back to uneditable format", exc + ) + else: + if req is not None: + return (req, True, []) + + logger.warning( + 'Could not determine repository location of %s', location + ) + comments = ['## !! Could not determine repository location'] + + return (None, False, comments) + + class FrozenRequirement(object): def __init__(self, name, req, editable, comments=()): + # type: (str, Union[str, Requirement], bool, Iterable[str]) -> None self.name = name self.req = req self.editable = editable self.comments = comments - _rev_re = re.compile(r'-r(\d+)$') - _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') - @classmethod - def _init_args_from_dist(cls, dist, dependency_links): - """ - Compute and return arguments (req, editable, comments) to pass to - FrozenRequirement.__init__(). - - This method is for use in FrozenRequirement.from_dist(). - """ - location = os.path.normcase(os.path.abspath(dist.location)) - comments = [] - from pipenv.patched.notpip._internal.vcs import vcs, get_src_requirement - if dist_is_editable(dist) and vcs.get_backend_name(location): - editable = True - try: - req = get_src_requirement(dist, location) - except InstallationError as exc: - logger.warning( - "Error when trying to get requirement for VCS system %s, " - "falling back to uneditable format", exc - ) - req = None - if req is None: - logger.warning( - 'Could not determine repository location of %s', location - ) - comments.append( - '## !! Could not determine repository location' - ) - req = dist.as_requirement() - editable = False - else: - editable = False + def from_dist(cls, dist): + # type: (Distribution) -> FrozenRequirement + req, editable, comments = get_requirement_info(dist) + if req is None: req = dist.as_requirement() - specs = req.specs - assert len(specs) == 1 and specs[0][0] in ["==", "==="], \ - 'Expected 1 spec with == or ===; specs = %r; dist = %r' % \ - (specs, dist) - version = specs[0][1] - ver_match = cls._rev_re.search(version) - date_match = cls._date_re.search(version) - if ver_match or date_match: - svn_backend = vcs.get_backend('svn') - if svn_backend: - svn_location = svn_backend().get_location( - dist, - dependency_links, - ) - if not svn_location: - logger.warning( - 'Warning: cannot find svn location for %s', req, - ) - comments.append( - '## FIXME: could not find svn URL in dependency_links ' - 'for this package:' - ) - else: - deprecated( - "SVN editable detection based on dependency links " - "will be dropped in the future.", - replacement=None, - gone_in="18.2", - issue=4187, - ) - comments.append( - '# Installing as editable to satisfy requirement %s:' % - req - ) - if ver_match: - rev = ver_match.group(1) - else: - rev = '{%s}' % date_match.group(1) - editable = True - egg_name = cls.egg_name(dist) - req = make_vcs_requirement_url(svn_location, rev, egg_name) - return (req, editable, comments) - - @classmethod - def from_dist(cls, dist, dependency_links): - args = cls._init_args_from_dist(dist, dependency_links) - return cls(dist.project_name, *args) - - @staticmethod - def egg_name(dist): - name = dist.egg_name() - match = re.search(r'-py\d\.\d$', name) - if match: - name = name[:match.start()] - return name + return cls(dist.project_name, req, editable, comments=comments) def __str__(self): req = self.req diff --git a/pipenv/patched/notpip/_internal/operations/prepare.py b/pipenv/patched/notpip/_internal/operations/prepare.py index d61270b4..018fca97 100644 --- a/pipenv/patched/notpip/_internal/operations/prepare.py +++ b/pipenv/patched/notpip/_internal/operations/prepare.py @@ -18,12 +18,21 @@ from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.hashes import MissingHashes from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import display_path, normalize_path, rmtree +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.vcs import vcs +if MYPY_CHECK_RUNNING: + from typing import Any, Optional # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.index import PackageFinder # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker # noqa: F401 + logger = logging.getLogger(__name__) def make_abstract_dist(req): + # type: (InstallRequirement) -> DistAbstraction """Factory to make an abstract dist object. Preconditions: Either an editable req with a source_dir, or satisfied_by or @@ -59,40 +68,40 @@ class DistAbstraction(object): """ def __init__(self, req): - self.req = req + # type: (InstallRequirement) -> None + self.req = req # type: InstallRequirement - def dist(self, finder): + def dist(self): + # type: () -> Any """Return a setuptools Dist object.""" - raise NotImplementedError(self.dist) + raise NotImplementedError def prep_for_dist(self, finder, build_isolation): + # type: (PackageFinder, bool) -> Any """Ensure that we can get a Dist for this requirement.""" - raise NotImplementedError(self.dist) + raise NotImplementedError class IsWheel(DistAbstraction): - def dist(self, finder): + def dist(self): + # type: () -> pkg_resources.Distribution return list(pkg_resources.find_distributions( self.req.source_dir))[0] def prep_for_dist(self, finder, build_isolation): + # type: (PackageFinder, bool) -> Any # FIXME:https://github.com/pypa/pip/issues/1112 pass class IsSDist(DistAbstraction): - def dist(self, finder): - dist = self.req.get_dist() - # FIXME: shouldn't be globally added. - if finder and dist.has_metadata('dependency_links.txt'): - finder.add_dependency_links( - dist.get_metadata_lines('dependency_links.txt') - ) - return dist + def dist(self): + return self.req.get_dist() def prep_for_dist(self, finder, build_isolation): + # type: (PackageFinder, bool) -> None # Prepare for building. We need to: # 1. Load pyproject.toml (if it exists) # 2. Set up the build environment @@ -100,43 +109,64 @@ class IsSDist(DistAbstraction): self.req.load_pyproject_toml() should_isolate = self.req.use_pep517 and build_isolation + def _raise_conflicts(conflicting_with, conflicting_reqs): + raise InstallationError( + "Some build dependencies for %s conflict with %s: %s." % ( + self.req, conflicting_with, ', '.join( + '%s is incompatible with %s' % (installed, wanted) + for installed, wanted in sorted(conflicting)))) + if should_isolate: # Isolate in a BuildEnvironment and install the build-time # requirements. self.req.build_env = BuildEnvironment() self.req.build_env.install_requirements( - finder, self.req.pyproject_requires, + finder, self.req.pyproject_requires, 'overlay', "Installing build dependencies" ) - missing = [] - if self.req.requirements_to_check: - check = self.req.requirements_to_check - missing = self.req.build_env.missing_requirements(check) + conflicting, missing = self.req.build_env.check_requirements( + self.req.requirements_to_check + ) + if conflicting: + _raise_conflicts("PEP 517/518 supported requirements", + conflicting) if missing: logger.warning( "Missing build requirements in pyproject.toml for %s.", self.req, ) logger.warning( - "The project does not specify a build backend, and pip " - "cannot fall back to setuptools without %s.", + "The project does not specify a build backend, and " + "pip cannot fall back to setuptools without %s.", " and ".join(map(repr, sorted(missing))) ) + # Install any extra build dependencies that the backend requests. + # This must be done in a second pass, as the pyproject.toml + # dependencies must be installed before we can call the backend. + with self.req.build_env: + # We need to have the env active when calling the hook. + self.req.spin_message = "Getting requirements to build wheel" + reqs = self.req.pep517_backend.get_requires_for_build_wheel() + conflicting, missing = self.req.build_env.check_requirements(reqs) + if conflicting: + _raise_conflicts("the backend dependencies", conflicting) + self.req.build_env.install_requirements( + finder, missing, 'normal', + "Installing backend dependencies" + ) - try: - self.req.run_egg_info() - except (OSError, TypeError): - self.req._correct_build_location() - self.req.run_egg_info() + self.req.prepare_metadata() self.req.assert_source_matches_version() class Installed(DistAbstraction): - def dist(self, finder): + def dist(self): + # type: () -> pkg_resources.Distribution return self.req.satisfied_by def prep_for_dist(self, finder, build_isolation): + # type: (PackageFinder, bool) -> Any pass @@ -144,8 +174,17 @@ class RequirementPreparer(object): """Prepares a Requirement """ - def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir, - progress_bar, build_isolation, req_tracker): + def __init__( + self, + build_dir, # type: str + download_dir, # type: Optional[str] + src_dir, # type: str + wheel_download_dir, # type: Optional[str] + progress_bar, # type: str + build_isolation, # type: bool + req_tracker # type: RequirementTracker + ): + # type: (...) -> None super(RequirementPreparer, self).__init__() self.src_dir = src_dir @@ -175,6 +214,7 @@ class RequirementPreparer(object): @property def _download_should_save(self): + # type: () -> bool # TODO: Modify to reduce indentation needed if self.download_dir: self.download_dir = expanduser(self.download_dir) @@ -187,8 +227,15 @@ class RequirementPreparer(object): % display_path(self.download_dir)) return False - def prepare_linked_requirement(self, req, session, finder, - upgrade_allowed, require_hashes): + def prepare_linked_requirement( + self, + req, # type: InstallRequirement + session, # type: PipSession + finder, # type: PackageFinder + upgrade_allowed, # type: bool + require_hashes # type: bool + ): + # type: (...) -> DistAbstraction """Prepare a requirement that would be obtained from req.link """ # TODO: Breakup into smaller functions @@ -209,6 +256,7 @@ class RequirementPreparer(object): # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req.source_dir` + # package unpacked in `req.source_dir` if os.path.exists(os.path.join(req.source_dir, 'setup.py')): rmtree(req.source_dir) req.populate_link(finder, upgrade_allowed, require_hashes) @@ -298,8 +346,14 @@ class RequirementPreparer(object): req.archive(self.download_dir) return abstract_dist - def prepare_editable_requirement(self, req, require_hashes, use_user_site, - finder): + def prepare_editable_requirement( + self, + req, # type: InstallRequirement + require_hashes, # type: bool + use_user_site, # type: bool + finder # type: PackageFinder + ): + # type: (...) -> DistAbstraction """Prepare an editable requirement """ assert req.editable, "cannot prepare a non-editable req as editable" @@ -327,6 +381,7 @@ class RequirementPreparer(object): return abstract_dist def prepare_installed_requirement(self, req, require_hashes, skip_reason): + # type: (InstallRequirement, bool, Optional[str]) -> DistAbstraction """Prepare an already-installed requirement """ assert req.satisfied_by, "req should have been satisfied but isn't" diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py index 182c1c88..4b6eb2bc 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py @@ -14,8 +14,15 @@ try: import pipenv.patched.notpip._internal.utils.glibc except ImportError: import pipenv.patched.notpip.utils.glibc - from pipenv.patched.notpip._internal.utils.compat import get_extension_suffixes +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Tuple, Callable, List, Optional, Union, Dict + ) + + Pep425Tag = Tuple[str, str, str] logger = logging.getLogger(__name__) @@ -23,6 +30,7 @@ _osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') def get_config_var(var): + # type: (str) -> Optional[str] try: return sysconfig.get_config_var(var) except IOError as e: # Issue #1074 @@ -31,6 +39,7 @@ def get_config_var(var): def get_abbr_impl(): + # type: () -> str """Return abbreviated implementation name.""" if hasattr(sys, 'pypy_version_info'): pyimpl = 'pp' @@ -44,6 +53,7 @@ def get_abbr_impl(): def get_impl_ver(): + # type: () -> str """Return implementation version.""" impl_ver = get_config_var("py_version_nodot") if not impl_ver or get_abbr_impl() == 'pp': @@ -52,17 +62,21 @@ def get_impl_ver(): def get_impl_version_info(): + # type: () -> Tuple[int, ...] """Return sys.version_info-like tuple for use in decrementing the minor version.""" if get_abbr_impl() == 'pp': # as per https://github.com/pypa/pip/issues/2882 - return (sys.version_info[0], sys.pypy_version_info.major, - sys.pypy_version_info.minor) + # attrs exist only on pypy + return (sys.version_info[0], + sys.pypy_version_info.major, # type: ignore + sys.pypy_version_info.minor) # type: ignore else: return sys.version_info[0], sys.version_info[1] def get_impl_tag(): + # type: () -> str """ Returns the Tag for this specific implementation. """ @@ -70,6 +84,7 @@ def get_impl_tag(): def get_flag(var, fallback, expected=True, warn=True): + # type: (str, Callable[..., bool], Union[bool, int], bool) -> bool """Use a fallback method for determining SOABI flags if the needed config var is unset or unavailable.""" val = get_config_var(var) @@ -82,6 +97,7 @@ def get_flag(var, fallback, expected=True, warn=True): def get_abi_tag(): + # type: () -> Optional[str] """Return the ABI tag based on SOABI (if available) or emulate SOABI (CPython 2, PyPy).""" soabi = get_config_var('SOABI') @@ -116,10 +132,12 @@ def get_abi_tag(): def _is_running_32bit(): + # type: () -> bool return sys.maxsize == 2147483647 def get_platform(): + # type: () -> str """Return our platform name 'win32', 'linux_x86_64'""" if sys.platform == 'darwin': # distutils.util.get_platform() returns the release based on the value @@ -146,6 +164,7 @@ def get_platform(): def is_manylinux1_compatible(): + # type: () -> bool # Only Linux, and only x86-64 / i686 if get_platform() not in {"linux_x86_64", "linux_i686"}: return False @@ -162,13 +181,33 @@ def is_manylinux1_compatible(): return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 5) +def is_manylinux2010_compatible(): + # type: () -> bool + # Only Linux, and only x86-64 / i686 + if get_platform() not in {"linux_x86_64", "linux_i686"}: + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux2010_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 6 uses glibc 2.12. + return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 12) + + def get_darwin_arches(major, minor, machine): + # type: (int, int, str) -> List[str] """Return a list of supported arches (including group arches) for the given major, minor and machine architecture of an macOS machine. """ arches = [] def _supports_arch(major, minor, arch): + # type: (int, int, str) -> bool # Looking at the application support for macOS versions in the chart # provided by https://en.wikipedia.org/wiki/OS_X#Versions it appears # our timeline looks roughly like: @@ -209,7 +248,7 @@ def get_darwin_arches(major, minor, machine): ("intel", ("x86_64", "i386")), ("fat64", ("x86_64", "ppc64")), ("fat32", ("x86_64", "i386", "ppc")), - ]) + ]) # type: Dict[str, Tuple[str, ...]] if _supports_arch(major, minor, machine): arches.append(machine) @@ -223,8 +262,24 @@ def get_darwin_arches(major, minor, machine): return arches -def get_supported(versions=None, noarch=False, platform=None, - impl=None, abi=None): +def get_all_minor_versions_as_strings(version_info): + # type: (Tuple[int, ...]) -> List[str] + versions = [] + major = version_info[:-1] + # Support all previous minor Python versions. + for minor in range(version_info[-1], -1, -1): + versions.append(''.join(map(str, major + (minor,)))) + return versions + + +def get_supported( + versions=None, # type: Optional[List[str]] + noarch=False, # type: bool + platform=None, # type: Optional[str] + impl=None, # type: Optional[str] + abi=None # type: Optional[str] +): + # type: (...) -> List[Pep425Tag] """Return a list of supported tags for each version specified in `versions`. @@ -241,16 +296,12 @@ def get_supported(versions=None, noarch=False, platform=None, # Versions must be given with respect to the preference if versions is None: - versions = [] version_info = get_impl_version_info() - major = version_info[:-1] - # Support all previous minor Python versions. - for minor in range(version_info[-1], -1, -1): - versions.append(''.join(map(str, major + (minor,)))) + versions = get_all_minor_versions_as_strings(version_info) impl = impl or get_abbr_impl() - abis = [] + abis = [] # type: List[str] abi = abi or get_abi_tag() if abi: @@ -267,6 +318,7 @@ def get_supported(versions=None, noarch=False, platform=None, if not noarch: arch = platform or get_platform() + arch_prefix, arch_sep, arch_suffix = arch.partition('_') if arch.startswith('macosx'): # support macosx-10.6-intel on macosx-10.9-x86_64 match = _osx_arch_pat.match(arch) @@ -280,8 +332,19 @@ def get_supported(versions=None, noarch=False, platform=None, else: # arch pattern didn't match (?!) arches = [arch] - elif platform is None and is_manylinux1_compatible(): - arches = [arch.replace('linux', 'manylinux1'), arch] + elif arch_prefix == 'manylinux2010': + # manylinux1 wheels run on most manylinux2010 systems with the + # exception of wheels depending on ncurses. PEP 571 states + # manylinux1 wheels should be considered manylinux2010 wheels: + # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels + arches = [arch, 'manylinux1' + arch_sep + arch_suffix] + elif platform is None: + arches = [] + if is_manylinux2010_compatible(): + arches.append('manylinux2010' + arch_sep + arch_suffix) + if is_manylinux1_compatible(): + arches.append('manylinux1' + arch_sep + arch_suffix) + arches.append(arch) else: arches = [arch] diff --git a/pipenv/patched/notpip/_internal/pyproject.py b/pipenv/patched/notpip/_internal/pyproject.py index a47e0f05..8845b2dc 100644 --- a/pipenv/patched/notpip/_internal/pyproject.py +++ b/pipenv/patched/notpip/_internal/pyproject.py @@ -2,20 +2,43 @@ from __future__ import absolute_import import io import os +import sys from pipenv.patched.notpip._vendor import pytoml, six from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any, Tuple, Optional, List # noqa: F401 def _is_list_of_str(obj): + # type: (Any) -> bool return ( isinstance(obj, list) and all(isinstance(item, six.string_types) for item in obj) ) -def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): +def make_pyproject_path(setup_py_dir): + # type: (str) -> str + path = os.path.join(setup_py_dir, 'pyproject.toml') + + # Python2 __file__ should not be unicode + if six.PY2 and isinstance(path, six.text_type): + path = path.encode(sys.getfilesystemencoding()) + + return path + + +def load_pyproject_toml( + use_pep517, # type: Optional[bool] + pyproject_toml, # type: str + setup_py, # type: str + req_name # type: str +): + # type: (...) -> Optional[Tuple[List[str], str, List[str]]] """Load the pyproject.toml file. Parameters: @@ -46,17 +69,20 @@ def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): build_system = None # The following cases must use PEP 517 - # We check for use_pep517 equalling False because that - # means the user explicitly requested --no-use-pep517 + # We check for use_pep517 being non-None and falsey because that means + # the user explicitly requested --no-use-pep517. The value 0 as + # opposed to False can occur when the value is provided via an + # environment variable or config file option (due to the quirk of + # strtobool() returning an integer in pip's configuration code). if has_pyproject and not has_setup: - if use_pep517 is False: + if use_pep517 is not None and not use_pep517: raise InstallationError( "Disabling PEP 517 processing is invalid: " "project does not have a setup.py" ) use_pep517 = True elif build_system and "build-backend" in build_system: - if use_pep517 is False: + if use_pep517 is not None and not use_pep517: raise InstallationError( "Disabling PEP 517 processing is invalid: " "project specifies a build backend of {} " @@ -85,11 +111,13 @@ def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): # section, or the user has no pyproject.toml, but has opted in # explicitly via --use-pep517. # In the absence of any explicit backend specification, we - # assume the setuptools backend, and require wheel and a version - # of setuptools that supports that backend. + # assume the setuptools backend that most closely emulates the + # traditional direct setup.py execution, and require wheel and + # a version of setuptools that supports that backend. + build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8.0", "wheel"], + "build-backend": "setuptools.build_meta:__legacy__", } # If we're using PEP 517, we have build system information (either @@ -123,22 +151,21 @@ def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): )) backend = build_system.get("build-backend") - check = [] + check = [] # type: List[str] if backend is None: # If the user didn't specify a backend, we assume they want to use # the setuptools backend. But we can't be sure they have included # a version of setuptools which supplies the backend, or wheel - # (which is neede by the backend) in their requirements. So we + # (which is needed by the backend) in their requirements. So we # make a note to check that those requirements are present once # we have set up the environment. - # TODO: Review this - it's quite a lot of work to check for a very - # specific case. The problem is, that case is potentially quite - # common - projects that adopted PEP 518 early for the ability to - # specify requirements to execute setup.py, but never considered - # needing to mention the build tools themselves. The original PEP - # 518 code had a similar check (but implemented in a different - # way). - backend = "setuptools.build_meta" - check = ["setuptools>=38.2.5", "wheel"] + # This is quite a lot of work to check for a very specific case. But + # the problem is, that case is potentially quite common - projects that + # adopted PEP 518 early for the ability to specify requirements to + # execute setup.py, but never considered needing to mention the build + # tools themselves. The original PEP 518 code had a similar check (but + # implemented in a different way). + backend = "setuptools.build_meta:__legacy__" + check = ["setuptools>=40.8.0", "wheel"] return (requires, backend, check) diff --git a/pipenv/patched/notpip/_internal/req/__init__.py b/pipenv/patched/notpip/_internal/req/__init__.py index 72aecb7c..51606fec 100644 --- a/pipenv/patched/notpip/_internal/req/__init__.py +++ b/pipenv/patched/notpip/_internal/req/__init__.py @@ -6,7 +6,10 @@ from .req_install import InstallRequirement from .req_set import RequirementSet from .req_file import parse_requirements from pipenv.patched.notpip._internal.utils.logging import indent_log +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +if MYPY_CHECK_RUNNING: + from typing import List, Sequence # noqa: F401 __all__ = [ "RequirementSet", "InstallRequirement", @@ -16,8 +19,13 @@ __all__ = [ logger = logging.getLogger(__name__) -def install_given_reqs(to_install, install_options, global_options=(), - *args, **kwargs): +def install_given_reqs( + to_install, # type: List[InstallRequirement] + install_options, # type: List[str] + global_options=(), # type: Sequence[str] + *args, **kwargs +): + # type: (...) -> List[InstallRequirement] """ Install everything in the given list. diff --git a/pipenv/patched/notpip/_internal/req/constructors.py b/pipenv/patched/notpip/_internal/req/constructors.py index 9fe28d88..a9d8221a 100644 --- a/pipenv/patched/notpip/_internal/req/constructors.py +++ b/pipenv/patched/notpip/_internal/req/constructors.py @@ -11,7 +11,6 @@ InstallRequirement. import logging import os import re -import traceback from pipenv.patched.notpip._vendor.packaging.markers import Marker from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement @@ -24,11 +23,20 @@ from pipenv.patched.notpip._internal.download import ( from pipenv.patched.notpip._internal.exceptions import InstallationError from pipenv.patched.notpip._internal.models.index import PyPI, TestPyPI from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.pyproject import make_pyproject_path from pipenv.patched.notpip._internal.req.req_install import InstallRequirement from pipenv.patched.notpip._internal.utils.misc import is_installable_dir +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.vcs import vcs from pipenv.patched.notpip._internal.wheel import Wheel +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Optional, Tuple, Set, Any, Union, Text, Dict, + ) + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + + __all__ = [ "install_req_from_editable", "install_req_from_line", "parse_editable" @@ -39,6 +47,7 @@ operators = Specifier._operators.keys() def _strip_extras(path): + # type: (str) -> Tuple[str, Optional[str]] m = re.match(r'^(.+)(\[[^\]]+\])$', path) extras = None if m: @@ -51,6 +60,7 @@ def _strip_extras(path): def parse_editable(editable_req): + # type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]] """Parses an editable requirement into: - a requirement name - an URL @@ -68,10 +78,18 @@ def parse_editable(editable_req): if os.path.isdir(url_no_extras): if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' not found." % - url_no_extras + msg = ( + 'File "setup.py" not found. Directory cannot be installed ' + 'in editable mode: {}'.format(os.path.abspath(url_no_extras)) ) + pyproject_path = make_pyproject_path(url_no_extras) + if os.path.isfile(pyproject_path): + msg += ( + '\n(A "pyproject.toml" file was found, but editable ' + 'mode currently requires a setup.py based build.)' + ) + raise InstallationError(msg) + # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) @@ -116,6 +134,7 @@ def parse_editable(editable_req): def deduce_helpful_msg(req): + # type: (str) -> str """Returns helpful msg in case requirements file does not exist, or cannot be parsed. @@ -136,7 +155,7 @@ def deduce_helpful_msg(req): " the packages specified within it." except RequirementParseError: logger.debug("Cannot parse '%s' as requirements \ - file" % (req), exc_info=1) + file" % (req), exc_info=True) else: msg += " File '%s' does not exist." % (req) return msg @@ -146,9 +165,15 @@ def deduce_helpful_msg(req): def install_req_from_editable( - editable_req, comes_from=None, isolated=False, options=None, - wheel_cache=None, constraint=False + editable_req, # type: str + comes_from=None, # type: Optional[str] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False # type: bool ): + # type: (...) -> InstallRequirement name, url, extras_override = parse_editable(editable_req) if url.startswith('file:'): source_dir = url_to_path(url) @@ -167,6 +192,7 @@ def install_req_from_editable( editable=True, link=Link(url), constraint=constraint, + use_pep517=use_pep517, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, @@ -175,9 +201,15 @@ def install_req_from_editable( def install_req_from_line( - name, comes_from=None, isolated=False, options=None, wheel_cache=None, - constraint=False + name, # type: str + comes_from=None, # type: Optional[Union[str, InstallRequirement]] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False # type: bool ): + # type: (...) -> InstallRequirement """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ @@ -186,24 +218,24 @@ def install_req_from_line( else: marker_sep = ';' if marker_sep in name: - name, markers = name.split(marker_sep, 1) - markers = markers.strip() - if not markers: + name, markers_as_string = name.split(marker_sep, 1) + markers_as_string = markers_as_string.strip() + if not markers_as_string: markers = None else: - markers = Marker(markers) + markers = Marker(markers_as_string) else: markers = None name = name.strip() - req = None + req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None - extras = None + extras_as_string = None if is_url(name): link = Link(name) else: - p, extras = _strip_extras(path) + p, extras_as_string = _strip_extras(path) looks_like_dir = os.path.isdir(p) and ( os.path.sep in name or (os.path.altsep is not None and os.path.altsep in name) or @@ -234,38 +266,41 @@ def install_req_from_line( # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename - req = "%s==%s" % (wheel.name, wheel.version) + req_as_string = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement - req = link.egg_fragment + req_as_string = link.egg_fragment # a requirement specifier else: - req = name + req_as_string = name - if extras: - extras = Requirement("placeholder" + extras.lower()).extras + if extras_as_string: + extras = Requirement("placeholder" + extras_as_string.lower()).extras else: extras = () - if req is not None: + if req_as_string is not None: try: - req = Requirement(req) + req = Requirement(req_as_string) except InvalidRequirement: - if os.path.sep in req: + if os.path.sep in req_as_string: add_msg = "It looks like a path." - add_msg += deduce_helpful_msg(req) - elif '=' in req and not any(op in req for op in operators): + add_msg += deduce_helpful_msg(req_as_string) + elif ('=' in req_as_string and + not any(op in req_as_string for op in operators)): add_msg = "= is not a valid operator. Did you mean == ?" else: - add_msg = traceback.format_exc() + add_msg = "" raise InstallationError( - "Invalid requirement: '%s'\n%s" % (req, add_msg) + "Invalid requirement: '%s'\n%s" % (req_as_string, add_msg) ) + else: + req = None return InstallRequirement( req, comes_from, link=link, markers=markers, - isolated=isolated, + use_pep517=use_pep517, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, constraint=constraint, @@ -273,11 +308,16 @@ def install_req_from_line( ) -def install_req_from_req( - req, comes_from=None, isolated=False, wheel_cache=None +def install_req_from_req_string( + req_string, # type: str + comes_from=None, # type: Optional[InstallRequirement] + isolated=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None # type: Optional[bool] ): + # type: (...) -> InstallRequirement try: - req = Requirement(req) + req = Requirement(req_string) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % req) @@ -294,5 +334,6 @@ def install_req_from_req( ) return InstallRequirement( - req, comes_from, isolated=isolated, wheel_cache=wheel_cache + req, comes_from, isolated=isolated, wheel_cache=wheel_cache, + use_pep517=use_pep517 ) diff --git a/pipenv/patched/notpip/_internal/req/req_file.py b/pipenv/patched/notpip/_internal/req/req_file.py index 5f23cd3a..30391cb5 100644 --- a/pipenv/patched/notpip/_internal/req/req_file.py +++ b/pipenv/patched/notpip/_internal/req/req_file.py @@ -19,6 +19,18 @@ from pipenv.patched.notpip._internal.exceptions import RequirementsFileParseErro from pipenv.patched.notpip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Iterator, Tuple, Optional, List, Callable, Text + ) + from pipenv.patched.notpip._internal.req import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + from pipenv.patched.notpip._internal.index import PackageFinder # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + + ReqFileLines = Iterator[Tuple[int, Text]] __all__ = ['parse_requirements'] @@ -43,24 +55,32 @@ SUPPORTED_OPTIONS = [ cmdoptions.no_binary, cmdoptions.only_binary, cmdoptions.pre, - cmdoptions.process_dependency_links, cmdoptions.trusted_host, cmdoptions.require_hashes, -] +] # type: List[Callable[..., optparse.Option]] # options to be passed to requirements SUPPORTED_OPTIONS_REQ = [ cmdoptions.install_options, cmdoptions.global_options, cmdoptions.hash, -] +] # type: List[Callable[..., optparse.Option]] # the 'dest' string values -SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ] +SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ] -def parse_requirements(filename, finder=None, comes_from=None, options=None, - session=None, constraint=False, wheel_cache=None): +def parse_requirements( + filename, # type: str + finder=None, # type: Optional[PackageFinder] + comes_from=None, # type: Optional[str] + options=None, # type: Optional[optparse.Values] + session=None, # type: Optional[PipSession] + constraint=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None # type: Optional[bool] +): + # type: (...) -> Iterator[InstallRequirement] """Parse a requirements file and yield InstallRequirement instances. :param filename: Path or url of requirements file. @@ -71,6 +91,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, :param constraint: If true, parsing a constraint file rather than requirements file. :param wheel_cache: Instance of pip.wheel.WheelCache + :param use_pep517: Value of the --use-pep517 option. """ if session is None: raise TypeError( @@ -87,18 +108,19 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, for line_number, line in lines_enum: req_iter = process_line(line, filename, line_number, finder, comes_from, options, session, wheel_cache, - constraint=constraint) + use_pep517=use_pep517, constraint=constraint) for req in req_iter: yield req def preprocess(content, options): + # type: (Text, Optional[optparse.Values]) -> ReqFileLines """Split, filter, and join lines, and return a line iterator :param content: the content of the requirements file :param options: cli options """ - lines_enum = enumerate(content.splitlines(), start=1) + lines_enum = enumerate(content.splitlines(), start=1) # type: ReqFileLines lines_enum = join_lines(lines_enum) lines_enum = ignore_comments(lines_enum) lines_enum = skip_regex(lines_enum, options) @@ -106,9 +128,19 @@ def preprocess(content, options): return lines_enum -def process_line(line, filename, line_number, finder=None, comes_from=None, - options=None, session=None, wheel_cache=None, - constraint=False): +def process_line( + line, # type: Text + filename, # type: str + line_number, # type: int + finder=None, # type: Optional[PackageFinder] + comes_from=None, # type: Optional[str] + options=None, # type: Optional[optparse.Values] + session=None, # type: Optional[PipSession] + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None, # type: Optional[bool] + constraint=False # type: bool +): + # type: (...) -> Iterator[InstallRequirement] """Process a single requirements line; This can result in creating/yielding requirements, or updating the finder. @@ -130,13 +162,15 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, defaults = parser.get_default_values() defaults.index_url = None if finder: - # `finder.format_control` will be updated during parsing defaults.format_control = finder.format_control args_str, options_str = break_args_options(line) + # Prior to 2.7.3, shlex cannot deal with unicode entries if sys.version_info < (2, 7, 3): - # Prior to 2.7.3, shlex cannot deal with unicode entries - options_str = options_str.encode('utf8') - opts, _ = parser.parse_args(shlex.split(options_str), defaults) + # https://github.com/python/mypy/issues/1174 + options_str = options_str.encode('utf8') # type: ignore + # https://github.com/python/mypy/issues/1174 + opts, _ = parser.parse_args( + shlex.split(options_str), defaults) # type: ignore # preserve for the nested code path line_comes_from = '%s %s (line %s)' % ( @@ -155,6 +189,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, req_options[dest] = opts.__dict__[dest] yield install_req_from_line( args_str, line_comes_from, constraint=constraint, + use_pep517=use_pep517, isolated=isolated, options=req_options, wheel_cache=wheel_cache ) @@ -163,6 +198,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, isolated = options.isolated_mode if options else False yield install_req_from_editable( opts.editables[0], comes_from=line_comes_from, + use_pep517=use_pep517, constraint=constraint, isolated=isolated, wheel_cache=wheel_cache ) @@ -183,11 +219,11 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, # do a join so relative paths work req_path = os.path.join(os.path.dirname(filename), req_path) # TODO: Why not use `comes_from='-r {} (line {})'` here as well? - parser = parse_requirements( + parsed_reqs = parse_requirements( req_path, finder, comes_from, options, session, constraint=nested_constraint, wheel_cache=wheel_cache ) - for req in parser: + for req in parsed_reqs: yield req # percolate hash-checking option upward @@ -214,14 +250,13 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, finder.find_links.append(value) if opts.pre: finder.allow_all_prereleases = True - if opts.process_dependency_links: - finder.process_dependency_links = True if opts.trusted_hosts: finder.secure_origins.extend( ("*", host, "*") for host in opts.trusted_hosts) def break_args_options(line): + # type: (Text) -> Tuple[str, Text] """Break up the line into an args and options string. We only want to shlex (and then optparse) the options, not the args. args can contain markers which are corrupted by shlex. @@ -235,10 +270,11 @@ def break_args_options(line): else: args.append(token) options.pop(0) - return ' '.join(args), ' '.join(options) + return ' '.join(args), ' '.join(options) # type: ignore def build_parser(line): + # type: (Text) -> optparse.OptionParser """ Return a parser for parsing requirement lines """ @@ -255,17 +291,20 @@ def build_parser(line): # add offending line msg = 'Invalid requirement: %s\n%s' % (line, msg) raise RequirementsFileParseError(msg) - parser.exit = parser_exit + # NOTE: mypy disallows assigning to a method + # https://github.com/python/mypy/issues/2427 + parser.exit = parser_exit # type: ignore return parser def join_lines(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """Joins a line ending in '\' with the previous line (except when following comments). The joined line takes on the index of the first line. """ primary_line_number = None - new_line = [] + new_line = [] # type: List[Text] for line_number, line in lines_enum: if not line.endswith('\\') or COMMENT_RE.match(line): if COMMENT_RE.match(line): @@ -290,6 +329,7 @@ def join_lines(lines_enum): def ignore_comments(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """ Strips comments and filter empty lines. """ @@ -301,6 +341,7 @@ def ignore_comments(lines_enum): def skip_regex(lines_enum, options): + # type: (ReqFileLines, Optional[optparse.Values]) -> ReqFileLines """ Skip lines that match '--skip-requirements-regex' pattern @@ -314,6 +355,7 @@ def skip_regex(lines_enum, options): def expand_env_variables(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """Replace all environment variables that can be retrieved via `os.getenv`. The only allowed format for environment variables defined in the diff --git a/pipenv/patched/notpip/_internal/req/req_install.py b/pipenv/patched/notpip/_internal/req/req_install.py index 3f32892c..fd3cead6 100644 --- a/pipenv/patched/notpip/_internal/req/req_install.py +++ b/pipenv/patched/notpip/_internal/req/req_install.py @@ -22,7 +22,7 @@ from pipenv.patched.notpip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) from pipenv.patched.notpip._internal.models.link import Link -from pipenv.patched.notpip._internal.pyproject import load_pyproject_toml +from pipenv.patched.notpip._internal.pyproject import load_pyproject_toml, make_pyproject_path from pipenv.patched.notpip._internal.req.req_uninstall import UninstallPathSet from pipenv.patched.notpip._internal.utils.compat import native_str from pipenv.patched.notpip._internal.utils.hashes import Hashes @@ -30,15 +30,28 @@ from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( _make_build_dir, ask_path_exists, backup_dir, call_subprocess, display_path, dist_in_site_packages, dist_in_usersite, ensure_dir, - get_installed_version, rmtree, + get_installed_version, redact_password_from_url, rmtree, ) from pipenv.patched.notpip._internal.utils.packaging import get_metadata from pipenv.patched.notpip._internal.utils.setuptools_build import SETUPTOOLS_SHIM from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import open_spinner from pipenv.patched.notpip._internal.vcs import vcs from pipenv.patched.notpip._internal.wheel import move_wheel_files +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Optional, Iterable, List, Union, Any, Text, Sequence, Dict + ) + from pipenv.patched.notpip._internal.build_env import BuildEnvironment # noqa: F401 + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + from pipenv.patched.notpip._internal.index import PackageFinder # noqa: F401 + from pipenv.patched.notpip._vendor.pkg_resources import Distribution # noqa: F401 + from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet # noqa: F401 + from pipenv.patched.notpip._vendor.packaging.markers import Marker # noqa: F401 + + logger = logging.getLogger(__name__) @@ -49,10 +62,23 @@ class InstallRequirement(object): installing the said requirement. """ - def __init__(self, req, comes_from, source_dir=None, editable=False, - link=None, update=True, markers=None, - isolated=False, options=None, wheel_cache=None, - constraint=False, extras=()): + def __init__( + self, + req, # type: Optional[Requirement] + comes_from, # type: Optional[Union[str, InstallRequirement]] + source_dir=None, # type: Optional[str] + editable=False, # type: bool + link=None, # type: Optional[Link] + update=True, # type: bool + markers=None, # type: Optional[Marker] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False, # type: bool + extras=() # type: Iterable[str] + ): + # type: (...) -> None assert req is None or isinstance(req, Requirement), req self.req = req self.comes_from = comes_from @@ -64,10 +90,10 @@ class InstallRequirement(object): self.editable = editable self._wheel_cache = wheel_cache - if link is not None: - self.link = self.original_link = link - else: - self.link = self.original_link = req and req.url and Link(req.url) + if link is None and req and req.url: + # PEP 508 URL requirement + link = Link(req.url) + self.link = self.original_link = link if extras: self.extras = extras @@ -77,11 +103,11 @@ class InstallRequirement(object): } else: self.extras = set() - if markers is not None: - self.markers = markers - else: - self.markers = req and req.marker - self._egg_info_path = None + if markers is None and req: + markers = req.marker + self.markers = markers + + self._egg_info_path = None # type: Optional[str] # This holds the pkg_resources.Distribution object if this requirement # is already available: self.satisfied_by = None @@ -92,11 +118,11 @@ class InstallRequirement(object): self._temp_build_dir = TempDirectory(kind="req-build") # Used to store the global directory where the _temp_build_dir should # have been created. Cf _correct_build_location method. - self._ideal_build_dir = None + self._ideal_build_dir = None # type: Optional[str] # True if the editable should be updated: self.update = update # Set to True after successful installation - self.install_succeeded = None + self.install_succeeded = None # type: Optional[bool] # UninstallPathSet of uninstalled distribution (for possible rollback) self.uninstalled_pathset = None self.options = options if options else {} @@ -105,32 +131,37 @@ class InstallRequirement(object): self.is_direct = False self.isolated = isolated - self.build_env = NoOpBuildEnvironment() + self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment + + # For PEP 517, the directory where we request the project metadata + # gets stored. We need this to pass to build_wheel, so the backend + # can ensure that the wheel matches the metadata (see the PEP for + # details). + self.metadata_directory = None # type: Optional[str] # The static build requirements (from pyproject.toml) - self.pyproject_requires = None + self.pyproject_requires = None # type: Optional[List[str]] # Build requirements that we will check are available - # TODO: We don't do this for --no-build-isolation. Should we? - self.requirements_to_check = [] + self.requirements_to_check = [] # type: List[str] # The PEP 517 backend we should use to build the project - self.pep517_backend = None + self.pep517_backend = None # type: Optional[Pep517HookCaller] # Are we using PEP 517 for this requirement? # After pyproject.toml has been loaded, the only valid values are True # and False. Before loading, None is valid (meaning "use the default"). # Setting an explicit value before loading pyproject.toml is supported, # but after loading this flag should be treated as read only. - self.use_pep517 = None + self.use_pep517 = use_pep517 def __str__(self): if self.req: s = str(self.req) if self.link: - s += ' from %s' % self.link.url + s += ' from %s' % redact_password_from_url(self.link.url) elif self.link: - s = self.link.url + s = redact_password_from_url(self.link.url) else: s = '' if self.satisfied_by is not None: @@ -149,6 +180,7 @@ class InstallRequirement(object): self.__class__.__name__, str(self), self.editable) def populate_link(self, finder, upgrade, require_hashes): + # type: (PackageFinder, bool, bool) -> None """Ensure that if a link can be found for this, that it is found. Note that self.link may still be None - if Upgrade is False and the @@ -171,16 +203,19 @@ class InstallRequirement(object): # Things that are valid for all kinds of requirements? @property def name(self): + # type: () -> Optional[str] if self.req is None: return None return native_str(pkg_resources.safe_name(self.req.name)) @property def specifier(self): + # type: () -> SpecifierSet return self.req.specifier @property def is_pinned(self): + # type: () -> bool """Return whether I am pinned to an exact version. For example, some-package==1.2 is pinned; some-package>1.2 is not. @@ -194,6 +229,7 @@ class InstallRequirement(object): return get_installed_version(self.name) def match_markers(self, extras_requested=None): + # type: (Optional[Iterable[str]]) -> bool if not extras_requested: # Provide an extra to safely evaluate the markers # without matching any extra @@ -207,6 +243,7 @@ class InstallRequirement(object): @property def has_hash_options(self): + # type: () -> bool """Return whether any known-good hashes are specified as options. These activate --require-hashes mode; hashes specified as part of a @@ -216,6 +253,7 @@ class InstallRequirement(object): return bool(self.options.get('hashes', {})) def hashes(self, trust_internet=True): + # type: (bool) -> Hashes """Return a hash-comparer that considers my option- and URL-based hashes to be known-good. @@ -237,6 +275,7 @@ class InstallRequirement(object): return Hashes(good_hashes) def from_path(self): + # type: () -> Optional[str] """Format a nice indicator to show where this "comes from" """ if self.req is None: @@ -252,6 +291,7 @@ class InstallRequirement(object): return s def build_location(self, build_dir): + # type: (str) -> Optional[str] assert build_dir is not None if self._temp_build_dir.path is not None: return self._temp_build_dir.path @@ -279,6 +319,7 @@ class InstallRequirement(object): return os.path.join(build_dir, name) def _correct_build_location(self): + # type: () -> None """Move self._temp_build_dir to self._ideal_build_dir/self.req.name For some requirements (e.g. a path to a directory), the name of the @@ -292,7 +333,8 @@ class InstallRequirement(object): return assert self.req is not None assert self._temp_build_dir.path - assert self._ideal_build_dir.path + assert (self._ideal_build_dir is not None and + self._ideal_build_dir.path) # type: ignore old_location = self._temp_build_dir.path self._temp_build_dir.path = None @@ -311,7 +353,16 @@ class InstallRequirement(object): self.source_dir = os.path.normpath(os.path.abspath(new_location)) self._egg_info_path = None + # Correct the metadata directory, if it exists + if self.metadata_directory: + old_meta = self.metadata_directory + rel = os.path.relpath(old_meta, start=old_location) + new_meta = os.path.join(new_location, rel) + new_meta = os.path.normpath(os.path.abspath(new_meta)) + self.metadata_directory = new_meta + def remove_temporary_source(self): + # type: () -> None """Remove the source files from this requirement, if they are marked for deletion""" if self.source_dir and os.path.exists( @@ -323,6 +374,7 @@ class InstallRequirement(object): self.build_env.cleanup() def check_if_exists(self, use_user_site): + # type: (bool) -> bool """Find an installed distribution that satisfies or conflicts with this requirement, and set self.satisfied_by or self.conflicts_with appropriately. @@ -366,11 +418,22 @@ class InstallRequirement(object): # Things valid for wheels @property def is_wheel(self): - return self.link and self.link.is_wheel + # type: () -> bool + if not self.link: + return False + return self.link.is_wheel - def move_wheel_files(self, wheeldir, root=None, home=None, prefix=None, - warn_script_location=True, use_user_site=False, - pycompile=True): + def move_wheel_files( + self, + wheeldir, # type: str + root=None, # type: Optional[str] + home=None, # type: Optional[str] + prefix=None, # type: Optional[str] + warn_script_location=True, # type: bool + use_user_site=False, # type: bool + pycompile=True # type: bool + ): + # type: (...) -> None move_wheel_files( self.name, self.req, wheeldir, user=use_user_site, @@ -385,12 +448,14 @@ class InstallRequirement(object): # Things valid for sdists @property def setup_py_dir(self): + # type: () -> str return os.path.join( self.source_dir, self.link and self.link.subdirectory_fragment or '') @property def setup_py(self): + # type: () -> str assert self.source_dir, "No source dir for %s" % self setup_py = os.path.join(self.setup_py_dir, 'setup.py') @@ -403,17 +468,13 @@ class InstallRequirement(object): @property def pyproject_toml(self): + # type: () -> str assert self.source_dir, "No source dir for %s" % self - pp_toml = os.path.join(self.setup_py_dir, 'pyproject.toml') - - # Python2 __file__ should not be unicode - if six.PY2 and isinstance(pp_toml, six.text_type): - pp_toml = pp_toml.encode(sys.getfilesystemencoding()) - - return pp_toml + return make_pyproject_path(self.setup_py_dir) def load_pyproject_toml(self): + # type: () -> None """Load the pyproject.toml file. After calling this routine, all of the attributes related to PEP 517 @@ -437,41 +498,36 @@ class InstallRequirement(object): self.pyproject_requires = requires self.pep517_backend = Pep517HookCaller(self.setup_py_dir, backend) - def run_egg_info(self): + # Use a custom function to call subprocesses + self.spin_message = "" + + def runner(cmd, cwd=None, extra_environ=None): + with open_spinner(self.spin_message) as spinner: + call_subprocess( + cmd, + cwd=cwd, + extra_environ=extra_environ, + show_stdout=False, + spinner=spinner + ) + self.spin_message = "" + + self.pep517_backend._subprocess_runner = runner + + def prepare_metadata(self): + # type: () -> None + """Ensure that project metadata is available. + + Under PEP 517, call the backend hook to prepare the metadata. + Under legacy processing, call setup.py egg-info. + """ assert self.source_dir - if self.name: - logger.debug( - 'Running setup.py (path:%s) egg_info for package %s', - self.setup_py, self.name, - ) - else: - logger.debug( - 'Running setup.py (path:%s) egg_info for package from %s', - self.setup_py, self.link, - ) with indent_log(): - script = SETUPTOOLS_SHIM % self.setup_py - sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) - base_cmd = [sys_executable, '-c', script] - if self.isolated: - base_cmd += ["--no-user-cfg"] - egg_info_cmd = base_cmd + ['egg_info'] - # We can't put the .egg-info files at the root, because then the - # source code will be mistaken for an installed egg, causing - # problems - if self.editable: - egg_base_option = [] + if self.use_pep517: + self.prepare_pep517_metadata() else: - egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') - ensure_dir(egg_info_dir) - egg_base_option = ['--egg-base', 'pip-egg-info'] - with self.build_env: - call_subprocess( - egg_info_cmd + egg_base_option, - cwd=self.setup_py_dir, - show_stdout=False, - command_desc='python setup.py egg_info') + self.run_egg_info() if not self.req: if isinstance(parse_version(self.metadata["Version"]), Version): @@ -490,15 +546,72 @@ class InstallRequirement(object): metadata_name = canonicalize_name(self.metadata["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( - 'Running setup.py (path:%s) egg_info for package %s ' + 'Generating metadata for package %s ' 'produced metadata for project name %s. Fix your ' '#egg=%s fragments.', - self.setup_py, self.name, metadata_name, self.name + self.name, metadata_name, self.name ) self.req = Requirement(metadata_name) + def prepare_pep517_metadata(self): + # type: () -> None + assert self.pep517_backend is not None + + metadata_dir = os.path.join( + self.setup_py_dir, + 'pip-wheel-metadata' + ) + ensure_dir(metadata_dir) + + with self.build_env: + # Note that Pep517HookCaller implements a fallback for + # prepare_metadata_for_build_wheel, so we don't have to + # consider the possibility that this hook doesn't exist. + backend = self.pep517_backend + self.spin_message = "Preparing wheel metadata" + distinfo_dir = backend.prepare_metadata_for_build_wheel( + metadata_dir + ) + + self.metadata_directory = os.path.join(metadata_dir, distinfo_dir) + + def run_egg_info(self): + # type: () -> None + if self.name: + logger.debug( + 'Running setup.py (path:%s) egg_info for package %s', + self.setup_py, self.name, + ) + else: + logger.debug( + 'Running setup.py (path:%s) egg_info for package from %s', + self.setup_py, self.link, + ) + script = SETUPTOOLS_SHIM % self.setup_py + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + base_cmd = [sys_executable, '-c', script] + if self.isolated: + base_cmd += ["--no-user-cfg"] + egg_info_cmd = base_cmd + ['egg_info'] + # We can't put the .egg-info files at the root, because then the + # source code will be mistaken for an installed egg, causing + # problems + if self.editable: + egg_base_option = [] # type: List[str] + else: + egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') + ensure_dir(egg_info_dir) + egg_base_option = ['--egg-base', 'pip-egg-info'] + with self.build_env: + call_subprocess( + egg_info_cmd + egg_base_option, + cwd=self.setup_py_dir, + show_stdout=False, + command_desc='python setup.py egg_info') + @property def egg_info_path(self): + # type: () -> str if self._egg_info_path is None: if self.editable: base = self.source_dir @@ -557,18 +670,31 @@ class InstallRequirement(object): return self._metadata def get_dist(self): - """Return a pkg_resources.Distribution built from self.egg_info_path""" - egg_info = self.egg_info_path.rstrip(os.path.sep) - base_dir = os.path.dirname(egg_info) - metadata = pkg_resources.PathMetadata(base_dir, egg_info) - dist_name = os.path.splitext(os.path.basename(egg_info))[0] - return pkg_resources.Distribution( - os.path.dirname(egg_info), + # type: () -> Distribution + """Return a pkg_resources.Distribution for this requirement""" + if self.metadata_directory: + base_dir, distinfo = os.path.split(self.metadata_directory) + metadata = pkg_resources.PathMetadata( + base_dir, self.metadata_directory + ) + dist_name = os.path.splitext(distinfo)[0] + typ = pkg_resources.DistInfoDistribution + else: + egg_info = self.egg_info_path.rstrip(os.path.sep) + base_dir = os.path.dirname(egg_info) + metadata = pkg_resources.PathMetadata(base_dir, egg_info) + dist_name = os.path.splitext(os.path.basename(egg_info))[0] + # https://github.com/python/mypy/issues/1174 + typ = pkg_resources.Distribution # type: ignore + + return typ( + base_dir, project_name=dist_name, metadata=metadata, ) def assert_source_matches_version(self): + # type: () -> None assert self.source_dir version = self.metadata['version'] if self.req.specifier and version not in self.req.specifier: @@ -587,6 +713,7 @@ class InstallRequirement(object): # For both source distributions and editables def ensure_has_source_dir(self, parent_dir): + # type: (str) -> str """Ensure that a source_dir is set. This will create a temporary build dir if the name of the requirement @@ -601,8 +728,13 @@ class InstallRequirement(object): return self.source_dir # For editable installations - def install_editable(self, install_options, - global_options=(), prefix=None): + def install_editable( + self, + install_options, # type: List[str] + global_options=(), # type: Sequence[str] + prefix=None # type: Optional[str] + ): + # type: (...) -> None logger.info('Running setup.py develop for %s', self.name) if self.isolated: @@ -614,8 +746,8 @@ class InstallRequirement(object): with indent_log(): # FIXME: should we do --install-headers here too? - sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) with self.build_env: + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) call_subprocess( [ sys_executable, @@ -633,6 +765,7 @@ class InstallRequirement(object): self.install_succeeded = True def update_editable(self, obtain=True): + # type: (bool) -> None if not self.link: logger.debug( "Cannot update repository at %s; repository location is " @@ -664,6 +797,7 @@ class InstallRequirement(object): # Top-level Actions def uninstall(self, auto_confirm=False, verbose=False, use_user_site=False): + # type: (bool, bool, bool) -> Optional[UninstallPathSet] """ Uninstall the distribution currently satisfying this requirement. @@ -678,7 +812,7 @@ class InstallRequirement(object): """ if not self.check_if_exists(use_user_site): logger.warning("Skipping %s as it is not installed.", self.name) - return + return None dist = self.satisfied_by or self.conflicts_with uninstalled_pathset = UninstallPathSet.from_dist(dist) @@ -693,9 +827,16 @@ class InstallRequirement(object): name = name.replace(os.path.sep, '/') return name + def _get_archive_name(self, path, parentdir, rootdir): + # type: (str, str, str) -> str + path = os.path.join(parentdir, path) + name = self._clean_zip_name(path, rootdir) + return self.name + '/' + name + # TODO: Investigate if this should be kept in InstallRequirement # Seems to be used only when VCS + downloads def archive(self, build_dir): + # type: (str) -> None assert self.source_dir create_archive = True archive_name = '%s-%s.zip' % (self.name, self.metadata["version"]) @@ -729,23 +870,35 @@ class InstallRequirement(object): if 'pip-egg-info' in dirnames: dirnames.remove('pip-egg-info') for dirname in dirnames: - dirname = os.path.join(dirpath, dirname) - name = self._clean_zip_name(dirname, dir) - zipdir = zipfile.ZipInfo(self.name + '/' + name + '/') + dir_arcname = self._get_archive_name(dirname, + parentdir=dirpath, + rootdir=dir) + zipdir = zipfile.ZipInfo(dir_arcname + '/') zipdir.external_attr = 0x1ED << 16 # 0o755 zip.writestr(zipdir, '') for filename in filenames: if filename == PIP_DELETE_MARKER_FILENAME: continue + file_arcname = self._get_archive_name(filename, + parentdir=dirpath, + rootdir=dir) filename = os.path.join(dirpath, filename) - name = self._clean_zip_name(filename, dir) - zip.write(filename, self.name + '/' + name) + zip.write(filename, file_arcname) zip.close() logger.info('Saved %s', display_path(archive_path)) - def install(self, install_options, global_options=None, root=None, - home=None, prefix=None, warn_script_location=True, - use_user_site=False, pycompile=True): + def install( + self, + install_options, # type: List[str] + global_options=None, # type: Optional[Sequence[str]] + root=None, # type: Optional[str] + home=None, # type: Optional[str] + prefix=None, # type: Optional[str] + warn_script_location=True, # type: bool + use_user_site=False, # type: bool + pycompile=True # type: bool + ): + # type: (...) -> None global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( @@ -775,7 +928,8 @@ class InstallRequirement(object): self.options.get('install_options', []) if self.isolated: - global_options = global_options + ["--no-user-cfg"] + # https://github.com/python/mypy/issues/1174 + global_options = global_options + ["--no-user-cfg"] # type: ignore with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') @@ -834,8 +988,15 @@ class InstallRequirement(object): with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') - def get_install_args(self, global_options, record_filename, root, prefix, - pycompile): + def get_install_args( + self, + global_options, # type: Sequence[str] + record_filename, # type: str + root, # type: Optional[str] + prefix, # type: Optional[str] + pycompile # type: bool + ): + # type: (...) -> List[str] sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) install_args = [sys_executable, "-u"] install_args.append('-c') diff --git a/pipenv/patched/notpip/_internal/req/req_set.py b/pipenv/patched/notpip/_internal/req/req_set.py index a65851ff..e7da5b71 100644 --- a/pipenv/patched/notpip/_internal/req/req_set.py +++ b/pipenv/patched/notpip/_internal/req/req_set.py @@ -5,29 +5,36 @@ from collections import OrderedDict from pipenv.patched.notpip._internal.exceptions import InstallationError from pipenv.patched.notpip._internal.utils.logging import indent_log +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.wheel import Wheel +if MYPY_CHECK_RUNNING: + from typing import Optional, List, Tuple, Dict, Iterable # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + + logger = logging.getLogger(__name__) class RequirementSet(object): def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): + # type: (bool, bool) -> None """Create a RequirementSet. """ - self.requirements = OrderedDict() + self.requirements = OrderedDict() # type: Dict[str, InstallRequirement] # noqa: E501 self.require_hashes = require_hashes self.check_supported_wheels = check_supported_wheels if ignore_compatibility: self.check_supported_wheels = False - self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False + self.ignore_compatibility = (check_supported_wheels is False or ignore_compatibility is True) # Mapping of alias: real_name - self.requirement_aliases = {} - self.unnamed_requirements = [] - self.successfully_downloaded = [] - self.reqs_to_cleanup = [] + self.requirement_aliases = {} # type: Dict[str, str] + self.unnamed_requirements = [] # type: List[InstallRequirement] + self.successfully_downloaded = [] # type: List[InstallRequirement] + self.reqs_to_cleanup = [] # type: List[InstallRequirement] def __str__(self): reqs = [req for req in self.requirements.values() @@ -42,8 +49,13 @@ class RequirementSet(object): return ('<%s object; %d requirement(s): %s>' % (self.__class__.__name__, len(reqs), reqs_str)) - def add_requirement(self, install_req, parent_req_name=None, - extras_requested=None): + def add_requirement( + self, + install_req, # type: InstallRequirement + parent_req_name=None, # type: Optional[str] + extras_requested=None # type: Optional[Iterable[str]] + ): + # type: (...) -> Tuple[List[InstallRequirement], Optional[InstallRequirement]] # noqa: E501 """Add install_req as a requirement to install. :param parent_req_name: The name of the requirement that needed this @@ -155,6 +167,7 @@ class RequirementSet(object): return [existing_req], existing_req def has_requirement(self, project_name): + # type: (str) -> bool name = project_name.lower() if (name in self.requirements and not self.requirements[name].constraint or @@ -165,10 +178,12 @@ class RequirementSet(object): @property def has_requirements(self): + # type: () -> List[InstallRequirement] return list(req for req in self.requirements.values() if not req.constraint) or self.unnamed_requirements def get_requirement(self, project_name): + # type: (str) -> InstallRequirement for name in project_name, project_name.lower(): if name in self.requirements: return self.requirements[name] @@ -177,6 +192,7 @@ class RequirementSet(object): pass def cleanup_files(self): + # type: () -> None """Clean up files, remove builds.""" logger.debug('Cleaning up...') with indent_log(): diff --git a/pipenv/patched/notpip/_internal/req/req_tracker.py b/pipenv/patched/notpip/_internal/req/req_tracker.py index 6e0201fe..d17a4187 100644 --- a/pipenv/patched/notpip/_internal/req/req_tracker.py +++ b/pipenv/patched/notpip/_internal/req/req_tracker.py @@ -7,6 +7,12 @@ import logging import os from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Set, Iterator # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.models.link import Link # noqa: F401 logger = logging.getLogger(__name__) @@ -14,6 +20,7 @@ logger = logging.getLogger(__name__) class RequirementTracker(object): def __init__(self): + # type: () -> None self._root = os.environ.get('PIP_REQ_TRACKER') if self._root is None: self._temp_dir = TempDirectory(delete=False, kind='req-tracker') @@ -23,7 +30,7 @@ class RequirementTracker(object): else: self._temp_dir = None logger.debug('Re-using requirements tracker %r', self._root) - self._entries = set() + self._entries = set() # type: Set[InstallRequirement] def __enter__(self): return self @@ -32,10 +39,12 @@ class RequirementTracker(object): self.cleanup() def _entry_path(self, link): + # type: (Link) -> str hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() return os.path.join(self._root, hashed) def add(self, req): + # type: (InstallRequirement) -> None link = req.link info = str(req) entry_path = self._entry_path(link) @@ -54,12 +63,14 @@ class RequirementTracker(object): logger.debug('Added %s to build tracker %r', req, self._root) def remove(self, req): + # type: (InstallRequirement) -> None link = req.link self._entries.remove(req) os.unlink(self._entry_path(link)) logger.debug('Removed %s from build tracker %r', req, self._root) def cleanup(self): + # type: () -> None for req in set(self._entries): self.remove(req) remove = self._temp_dir is not None @@ -71,6 +82,7 @@ class RequirementTracker(object): @contextlib.contextmanager def track(self, req): + # type: (InstallRequirement) -> Iterator[None] self.add(req) yield self.remove(req) diff --git a/pipenv/patched/notpip/_internal/req/req_uninstall.py b/pipenv/patched/notpip/_internal/req/req_uninstall.py index 4cd15d84..ce80e6dc 100644 --- a/pipenv/patched/notpip/_internal/req/req_uninstall.py +++ b/pipenv/patched/notpip/_internal/req/req_uninstall.py @@ -15,9 +15,9 @@ from pipenv.patched.notpip._internal.utils.compat import WINDOWS, cache_from_sou from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local, - normalize_path, renames, + normalize_path, renames, rmtree, ) -from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory logger = logging.getLogger(__name__) @@ -86,16 +86,54 @@ def compact(paths): sep = os.path.sep short_paths = set() for path in sorted(paths, key=len): - should_add = any( + should_skip = any( path.startswith(shortpath.rstrip("*")) and path[len(shortpath.rstrip("*").rstrip(sep))] == sep for shortpath in short_paths ) - if not should_add: + if not should_skip: short_paths.add(path) return short_paths +def compress_for_rename(paths): + """Returns a set containing the paths that need to be renamed. + + This set may include directories when the original sequence of paths + included every file on disk. + """ + case_map = dict((os.path.normcase(p), p) for p in paths) + remaining = set(case_map) + unchecked = sorted(set(os.path.split(p)[0] + for p in case_map.values()), key=len) + wildcards = set() + + def norm_join(*a): + return os.path.normcase(os.path.join(*a)) + + for root in unchecked: + if any(os.path.normcase(root).startswith(w) + for w in wildcards): + # This directory has already been handled. + continue + + all_files = set() + all_subdirs = set() + for dirname, subdirs, files in os.walk(root): + all_subdirs.update(norm_join(root, dirname, d) + for d in subdirs) + all_files.update(norm_join(root, dirname, f) + for f in files) + # If all the files we found are in our remaining set of files to + # remove, then remove them from the latter set and add a wildcard + # for the directory. + if not (all_files - remaining): + remaining.difference_update(all_files) + wildcards.add(root + os.sep) + + return set(map(case_map.__getitem__, remaining)) | wildcards + + def compress_for_output_listing(paths): """Returns a tuple of 2 sets of which paths to display to user @@ -145,6 +183,111 @@ def compress_for_output_listing(paths): return will_remove, will_skip +class StashedUninstallPathSet(object): + """A set of file rename operations to stash files while + tentatively uninstalling them.""" + def __init__(self): + # Mapping from source file root to [Adjacent]TempDirectory + # for files under that directory. + self._save_dirs = {} + # (old path, new path) tuples for each move that may need + # to be undone. + self._moves = [] + + def _get_directory_stash(self, path): + """Stashes a directory. + + Directories are stashed adjacent to their original location if + possible, or else moved/copied into the user's temp dir.""" + + try: + save_dir = AdjacentTempDirectory(path) + save_dir.create() + except OSError: + save_dir = TempDirectory(kind="uninstall") + save_dir.create() + self._save_dirs[os.path.normcase(path)] = save_dir + + return save_dir.path + + def _get_file_stash(self, path): + """Stashes a file. + + If no root has been provided, one will be created for the directory + in the user's temp directory.""" + path = os.path.normcase(path) + head, old_head = os.path.dirname(path), None + save_dir = None + + while head != old_head: + try: + save_dir = self._save_dirs[head] + break + except KeyError: + pass + head, old_head = os.path.dirname(head), head + else: + # Did not find any suitable root + head = os.path.dirname(path) + save_dir = TempDirectory(kind='uninstall') + save_dir.create() + self._save_dirs[head] = save_dir + + relpath = os.path.relpath(path, head) + if relpath and relpath != os.path.curdir: + return os.path.join(save_dir.path, relpath) + return save_dir.path + + def stash(self, path): + """Stashes the directory or file and returns its new location. + """ + if os.path.isdir(path): + new_path = self._get_directory_stash(path) + else: + new_path = self._get_file_stash(path) + + self._moves.append((path, new_path)) + if os.path.isdir(path) and os.path.isdir(new_path): + # If we're moving a directory, we need to + # remove the destination first or else it will be + # moved to inside the existing directory. + # We just created new_path ourselves, so it will + # be removable. + os.rmdir(new_path) + renames(path, new_path) + return new_path + + def commit(self): + """Commits the uninstall by removing stashed files.""" + for _, save_dir in self._save_dirs.items(): + save_dir.cleanup() + self._moves = [] + self._save_dirs = {} + + def rollback(self): + """Undoes the uninstall by moving stashed files back.""" + for p in self._moves: + logging.info("Moving to %s\n from %s", *p) + + for new_path, path in self._moves: + try: + logger.debug('Replacing %s from %s', new_path, path) + if os.path.isfile(new_path): + os.unlink(new_path) + elif os.path.isdir(new_path): + rmtree(new_path) + renames(path, new_path) + except OSError as ex: + logger.error("Failed to restore %s", new_path) + logger.debug("Exception: %s", ex) + + self.commit() + + @property + def can_rollback(self): + return bool(self._moves) + + class UninstallPathSet(object): """A set of file paths to be removed in the uninstallation of a requirement.""" @@ -153,8 +296,7 @@ class UninstallPathSet(object): self._refuse = set() self.pth = {} self.dist = dist - self.save_dir = TempDirectory(kind="uninstall") - self._moved_paths = [] + self._moved_paths = StashedUninstallPathSet() def _permitted(self, path): """ @@ -192,11 +334,6 @@ class UninstallPathSet(object): else: self._refuse.add(pth_file) - def _stash(self, path): - return os.path.join( - self.save_dir.path, os.path.splitdrive(path)[1].lstrip(os.path.sep) - ) - def remove(self, auto_confirm=False, verbose=False): """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" @@ -215,13 +352,14 @@ class UninstallPathSet(object): with indent_log(): if auto_confirm or self._allowed_to_proceed(verbose): - self.save_dir.create() + moved = self._moved_paths - for path in sorted(compact(self.paths)): - new_path = self._stash(path) + for_rename = compress_for_rename(self.paths) + + for path in sorted(compact(for_rename)): + moved.stash(path) logger.debug('Removing file or directory %s', path) - self._moved_paths.append(path) - renames(path, new_path) + for pth in self.pth.values(): pth.remove() @@ -251,29 +389,27 @@ class UninstallPathSet(object): _display('Would remove:', will_remove) _display('Would not remove (might be manually added):', will_skip) _display('Would not remove (outside of prefix):', self._refuse) + if verbose: + _display('Will actually move:', compress_for_rename(self.paths)) return ask('Proceed (y/n)? ', ('y', 'n')) == 'y' def rollback(self): """Rollback the changes previously made by remove().""" - if self.save_dir.path is None: + if not self._moved_paths.can_rollback: logger.error( "Can't roll back %s; was not uninstalled", self.dist.project_name, ) return False logger.info('Rolling back uninstall of %s', self.dist.project_name) - for path in self._moved_paths: - tmp_path = self._stash(path) - logger.debug('Replacing %s', path) - renames(tmp_path, path) + self._moved_paths.rollback() for pth in self.pth.values(): pth.rollback() def commit(self): """Remove temporary save dir: rollback will no longer be possible.""" - self.save_dir.cleanup() - self._moved_paths = [] + self._moved_paths.commit() @classmethod def from_dist(cls, dist): diff --git a/pipenv/patched/notpip/_internal/resolve.py b/pipenv/patched/notpip/_internal/resolve.py index b0d096f9..79e4b409 100644 --- a/pipenv/patched/notpip/_internal/resolve.py +++ b/pipenv/patched/notpip/_internal/resolve.py @@ -18,10 +18,22 @@ from pipenv.patched.notpip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors, UnsupportedPythonVersion, ) -from pipenv.patched.notpip._internal.req.constructors import install_req_from_req +from pipenv.patched.notpip._internal.req.constructors import install_req_from_req_string from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import dist_in_usersite, ensure_dir from pipenv.patched.notpip._internal.utils.packaging import check_dist_requires_python +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, DefaultDict, List, Set # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.index import PackageFinder # noqa: F401 + from pipenv.patched.notpip._internal.req.req_set import RequirementSet # noqa: F401 + from pipenv.patched.notpip._internal.operations.prepare import ( # noqa: F401 + DistAbstraction, RequirementPreparer + ) + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 logger = logging.getLogger(__name__) @@ -33,9 +45,23 @@ class Resolver(object): _allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"} - def __init__(self, preparer, session, finder, wheel_cache, use_user_site, - ignore_dependencies, ignore_installed, ignore_requires_python, - force_reinstall, isolated, upgrade_strategy, ignore_compatibility=False): + def __init__( + self, + preparer, # type: RequirementPreparer + session, # type: PipSession + finder, # type: PackageFinder + wheel_cache, # type: Optional[WheelCache] + use_user_site, # type: bool + ignore_dependencies, # type: bool + ignore_installed, # type: bool + ignore_requires_python, # type: bool + force_reinstall, # type: bool + isolated, # type: bool + upgrade_strategy, # type: str + use_pep517=None, # type: Optional[bool] + ignore_compatibility=False, # type: bool + ): + # type: (...) -> None super(Resolver, self).__init__() assert upgrade_strategy in self._allowed_strategies @@ -47,7 +73,8 @@ class Resolver(object): # information about both sdist and wheels transparently. self.wheel_cache = wheel_cache - self.require_hashes = None # This is set in resolve + # This is set in resolve + self.require_hashes = None # type: Optional[bool] self.upgrade_strategy = upgrade_strategy self.force_reinstall = force_reinstall @@ -57,13 +84,16 @@ class Resolver(object): self.ignore_requires_python = ignore_requires_python self.ignore_compatibility = ignore_compatibility self.use_user_site = use_user_site + self.use_pep517 = use_pep517 self.requires_python = None if self.ignore_compatibility: self.ignore_requires_python = True - self._discovered_dependencies = defaultdict(list) + self._discovered_dependencies = \ + defaultdict(list) # type: DefaultDict[str, List] def resolve(self, requirement_set): + # type: (RequirementSet) -> None """Resolve what operations need to be done As a side-effect of this method, the packages (and their dependencies) @@ -98,7 +128,7 @@ class Resolver(object): # exceptions cannot be checked ahead of time, because # req.populate_link() needs to be called before we can make decisions # based on link type. - discovered_reqs = [] + discovered_reqs = [] # type: List[InstallRequirement] hash_errors = HashErrors() for req in chain(root_reqs, discovered_reqs): try: @@ -113,6 +143,7 @@ class Resolver(object): raise hash_errors def _is_upgrade_allowed(self, req): + # type: (InstallRequirement) -> bool if self.upgrade_strategy == "to-satisfy-only": return False elif self.upgrade_strategy == "eager": @@ -122,6 +153,7 @@ class Resolver(object): return req.is_direct def _set_req_to_reinstall(self, req): + # type: (InstallRequirement) -> None """ Set a requirement to be installed. """ @@ -133,6 +165,7 @@ class Resolver(object): # XXX: Stop passing requirement_set for options def _check_skip_installed(self, req_to_install): + # type: (InstallRequirement) -> Optional[str] """Check if req_to_install should be skipped. This will check if the req is installed, and whether we should upgrade @@ -185,6 +218,7 @@ class Resolver(object): return None def _get_abstract_dist_for(self, req): + # type: (InstallRequirement) -> DistAbstraction """Takes a InstallRequirement and returns a single AbstractDist \ representing a prepared variant of the same. """ @@ -241,7 +275,13 @@ class Resolver(object): return abstract_dist - def _resolve_one(self, requirement_set, req_to_install, ignore_requires_python=False): + def _resolve_one( + self, + requirement_set, # type: RequirementSet + req_to_install, # type: InstallRequirement + ignore_requires_python=False # type: bool + ): + # type: (...) -> List[InstallRequirement] """Prepare a single requirements file. :return: A list of additional InstallRequirements to also install. @@ -260,11 +300,11 @@ class Resolver(object): abstract_dist = self._get_abstract_dist_for(req_to_install) # Parse and return dependencies - dist = abstract_dist.dist(self.finder) + dist = abstract_dist.dist() try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: - if self.ignore_requires_python or self.ignore_compatibility: + if self.ignore_requires_python or ignore_requires_python or self.ignore_compatibility: logger.warning(err.args[0]) else: raise @@ -275,14 +315,16 @@ class Resolver(object): except TypeError: self.requires_python = None - more_reqs = [] + + more_reqs = [] # type: List[InstallRequirement] def add_req(subreq, extras_requested): - sub_install_req = install_req_from_req( + sub_install_req = install_req_from_req_string( str(subreq), req_to_install, isolated=self.isolated, wheel_cache=self.wheel_cache, + use_pep517=self.use_pep517 ) parent_req_name = req_to_install.name to_scan_again, add_to_parent = requirement_set.add_requirement( @@ -300,10 +342,10 @@ class Resolver(object): # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): - # 'unnamed' requirements will get added here available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) + # 'unnamed' requirements will get added here req_to_install.is_direct = True requirement_set.add_requirement( req_to_install, parent_req_name=None, @@ -335,11 +377,12 @@ class Resolver(object): for available in available_requested: if hasattr(dist, '_DistInfoDistribution__dep_map'): for req in dist._DistInfoDistribution__dep_map[available]: - req = install_req_from_req( + req = InstallRequirement( str(req), req_to_install, isolated=self.isolated, wheel_cache=self.wheel_cache, + use_pep517=None ) more_reqs.append(req) @@ -353,6 +396,7 @@ class Resolver(object): return more_reqs def get_installation_order(self, req_set): + # type: (RequirementSet) -> List[InstallRequirement] """Create the installation order. The installation order is topological - requirements are installed @@ -363,7 +407,7 @@ class Resolver(object): # installs the user specified things in the order given, except when # dependencies must come earlier to achieve topological order. order = [] - ordered_reqs = set() + ordered_reqs = set() # type: Set[InstallRequirement] def schedule(req): if req.satisfied_by or req in ordered_reqs: diff --git a/pipenv/patched/notpip/_internal/utils/appdirs.py b/pipenv/patched/notpip/_internal/utils/appdirs.py index e8e14526..9ce3a1b3 100644 --- a/pipenv/patched/notpip/_internal/utils/appdirs.py +++ b/pipenv/patched/notpip/_internal/utils/appdirs.py @@ -10,9 +10,16 @@ import sys from pipenv.patched.notpip._vendor.six import PY2, text_type from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + List, Union + ) def user_cache_dir(appname): + # type: (str) -> str r""" Return full path to the user-specific cache dir for this application. @@ -61,6 +68,7 @@ def user_cache_dir(appname): def user_data_dir(appname, roaming=False): + # type: (str, bool) -> str r""" Return full path to the user-specific data dir for this application. @@ -113,6 +121,7 @@ def user_data_dir(appname, roaming=False): def user_config_dir(appname, roaming=True): + # type: (str, bool) -> str """Return full path to the user-specific config dir for this application. "appname" is the name of application. @@ -146,6 +155,7 @@ def user_config_dir(appname, roaming=True): # for the discussion regarding site_config_dirs locations # see def site_config_dirs(appname): + # type: (str) -> List[str] r"""Return a list of potential user-shared config dirs for this application. "appname" is the name of application. @@ -186,6 +196,7 @@ def site_config_dirs(appname): # -- Windows support functions -- def _get_win_folder_from_registry(csidl_name): + # type: (str) -> str """ This is a fallback technique at best. I'm not sure if using the registry for this guarantees us the correct answer for all CSIDL_* @@ -208,6 +219,7 @@ def _get_win_folder_from_registry(csidl_name): def _get_win_folder_with_ctypes(csidl_name): + # type: (str) -> str csidl_const = { "CSIDL_APPDATA": 26, "CSIDL_COMMON_APPDATA": 35, diff --git a/pipenv/patched/notpip/_internal/utils/compat.py b/pipenv/patched/notpip/_internal/utils/compat.py index 483bfdc8..1dad56b0 100644 --- a/pipenv/patched/notpip/_internal/utils/compat.py +++ b/pipenv/patched/notpip/_internal/utils/compat.py @@ -11,6 +11,11 @@ import sys from pipenv.patched.notpip._vendor.six import text_type +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Tuple, Text # noqa: F401 + try: import ipaddress except ImportError: @@ -18,8 +23,8 @@ except ImportError: from pipenv.patched.notpip._vendor import ipaddress # type: ignore except ImportError: import ipaddr as ipaddress # type: ignore - ipaddress.ip_address = ipaddress.IPAddress - ipaddress.ip_network = ipaddress.IPNetwork + ipaddress.ip_address = ipaddress.IPAddress # type: ignore + ipaddress.ip_network = ipaddress.IPNetwork # type: ignore __all__ = [ @@ -68,6 +73,7 @@ else: def console_to_str(data): + # type: (bytes) -> Text """Return a string, safe for output, of subprocess output. We assume the data is in the locale preferred encoding. @@ -88,13 +94,13 @@ def console_to_str(data): # Now try to decode the data - if we fail, warn the user and # decode with replacement. try: - s = data.decode(encoding) + decoded_data = data.decode(encoding) except UnicodeDecodeError: logger.warning( "Subprocess output does not appear to be encoded as %s", encoding, ) - s = data.decode(encoding, errors=backslashreplace_decode) + decoded_data = data.decode(encoding, errors=backslashreplace_decode) # Make sure we can print the output, by encoding it to the output # encoding with replacement of unencodable characters, and then @@ -112,20 +118,25 @@ def console_to_str(data): "encoding", None) if output_encoding: - s = s.encode(output_encoding, errors="backslashreplace") - s = s.decode(output_encoding) + output_encoded = decoded_data.encode( + output_encoding, + errors="backslashreplace" + ) + decoded_data = output_encoded.decode(output_encoding) - return s + return decoded_data if sys.version_info >= (3,): def native_str(s, replace=False): + # type: (str, bool) -> str if isinstance(s, bytes): return s.decode('utf-8', 'replace' if replace else 'strict') return s else: def native_str(s, replace=False): + # type: (str, bool) -> str # Replace is ignored -- unicode to UTF-8 can't fail if isinstance(s, text_type): return s.encode('utf-8') @@ -133,6 +144,7 @@ else: def get_path_uid(path): + # type: (str) -> int """ Return path's uid. @@ -174,6 +186,7 @@ else: def expanduser(path): + # type: (str) -> str """ Expand ~ and ~user constructions. @@ -199,6 +212,7 @@ WINDOWS = (sys.platform.startswith("win") or def samefile(file1, file2): + # type: (str, str) -> bool """Provide an alternative for os.path.samefile on Windows/Python2""" if hasattr(os.path, 'samefile'): return os.path.samefile(file1, file2) @@ -210,13 +224,15 @@ def samefile(file1, file2): if hasattr(shutil, 'get_terminal_size'): def get_terminal_size(): + # type: () -> Tuple[int, int] """ Returns a tuple (x, y) representing the width(x) and the height(y) in characters of the terminal window. """ - return tuple(shutil.get_terminal_size()) + return tuple(shutil.get_terminal_size()) # type: ignore else: def get_terminal_size(): + # type: () -> Tuple[int, int] """ Returns a tuple (x, y) representing the width(x) and the height(y) in characters of the terminal window. diff --git a/pipenv/patched/notpip/_internal/utils/deprecation.py b/pipenv/patched/notpip/_internal/utils/deprecation.py index b140ac71..2e309ec2 100644 --- a/pipenv/patched/notpip/_internal/utils/deprecation.py +++ b/pipenv/patched/notpip/_internal/utils/deprecation.py @@ -41,6 +41,7 @@ def _showwarning(message, category, filename, lineno, file=None, line=None): def install_warning_logger(): + # type: () -> None # Enable our Deprecation Warnings warnings.simplefilter("default", PipDeprecationWarning, append=True) diff --git a/pipenv/patched/notpip/_internal/utils/encoding.py b/pipenv/patched/notpip/_internal/utils/encoding.py index 56f60361..f03fc901 100644 --- a/pipenv/patched/notpip/_internal/utils/encoding.py +++ b/pipenv/patched/notpip/_internal/utils/encoding.py @@ -3,6 +3,11 @@ import locale import re import sys +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Tuple, Text # noqa: F401 + BOMS = [ (codecs.BOM_UTF8, 'utf8'), (codecs.BOM_UTF16, 'utf16'), @@ -11,12 +16,13 @@ BOMS = [ (codecs.BOM_UTF32, 'utf32'), (codecs.BOM_UTF32_BE, 'utf32-be'), (codecs.BOM_UTF32_LE, 'utf32-le'), -] +] # type: List[Tuple[bytes, Text]] ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)') def auto_decode(data): + # type: (bytes) -> Text """Check a bytes string for a BOM to correctly detect the encoding Fallback to locale.getpreferredencoding(False) like open() on Python3""" diff --git a/pipenv/patched/notpip/_internal/utils/filesystem.py b/pipenv/patched/notpip/_internal/utils/filesystem.py index e8d6a2bb..d4aae97d 100644 --- a/pipenv/patched/notpip/_internal/utils/filesystem.py +++ b/pipenv/patched/notpip/_internal/utils/filesystem.py @@ -5,6 +5,7 @@ from pipenv.patched.notpip._internal.utils.compat import get_path_uid def check_path_owner(path): + # type: (str) -> bool # If we don't have a way to check the effective uid of this process, then # we'll just assume that we own the directory. if not hasattr(os, "geteuid"): @@ -26,3 +27,4 @@ def check_path_owner(path): return os.access(path, os.W_OK) else: previous, path = path, os.path.dirname(path) + return False # assume we don't own the path diff --git a/pipenv/patched/notpip/_internal/utils/glibc.py b/pipenv/patched/notpip/_internal/utils/glibc.py index ebcfc5be..e2b6d505 100644 --- a/pipenv/patched/notpip/_internal/utils/glibc.py +++ b/pipenv/patched/notpip/_internal/utils/glibc.py @@ -4,8 +4,14 @@ import ctypes import re import warnings +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple # noqa: F401 + def glibc_version_string(): + # type: () -> Optional[str] "Returns glibc version string, or None if not using glibc." # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen @@ -32,6 +38,7 @@ def glibc_version_string(): # Separated out from have_compatible_glibc for easier unit testing def check_glibc_version(version_str, required_major, minimum_minor): + # type: (str, int, int) -> bool # Parse string and check against requested version. # # We use a regexp instead of str.split because we want to discard any @@ -48,7 +55,8 @@ def check_glibc_version(version_str, required_major, minimum_minor): def have_compatible_glibc(required_major, minimum_minor): - version_str = glibc_version_string() + # type: (int, int) -> bool + version_str = glibc_version_string() # type: Optional[str] if version_str is None: return False return check_glibc_version(version_str, required_major, minimum_minor) @@ -72,6 +80,7 @@ def have_compatible_glibc(required_major, minimum_minor): # misleading. Solution: instead of using platform, use our code that actually # works. def libc_ver(): + # type: () -> Tuple[str, str] """Try to determine the glibc version Returns a tuple of strings (lib, version) which default to empty strings diff --git a/pipenv/patched/notpip/_internal/utils/hashes.py b/pipenv/patched/notpip/_internal/utils/hashes.py index 818f5505..55cb8411 100644 --- a/pipenv/patched/notpip/_internal/utils/hashes.py +++ b/pipenv/patched/notpip/_internal/utils/hashes.py @@ -8,6 +8,18 @@ from pipenv.patched.notpip._internal.exceptions import ( HashMismatch, HashMissing, InstallationError, ) from pipenv.patched.notpip._internal.utils.misc import read_chunks +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Dict, List, BinaryIO, NoReturn, Iterator + ) + from pipenv.patched.notpip._vendor.six import PY3 + if PY3: + from hashlib import _Hash # noqa: F401 + else: + from hashlib import _hash as _Hash # noqa: F401 + # The recommended hash algo of the moment. Change this whenever the state of # the art changes; it won't hurt backward compatibility. @@ -25,6 +37,7 @@ class Hashes(object): """ def __init__(self, hashes=None): + # type: (Dict[str, List[str]]) -> None """ :param hashes: A dict of algorithm names pointing to lists of allowed hex digests @@ -32,6 +45,7 @@ class Hashes(object): self._allowed = {} if hashes is None else hashes def check_against_chunks(self, chunks): + # type: (Iterator[bytes]) -> None """Check good hashes against ones built from iterable of chunks of data. @@ -55,9 +69,11 @@ class Hashes(object): self._raise(gots) def _raise(self, gots): + # type: (Dict[str, _Hash]) -> NoReturn raise HashMismatch(self._allowed, gots) def check_against_file(self, file): + # type: (BinaryIO) -> None """Check good hashes against a file-like object Raise HashMismatch if none match. @@ -66,14 +82,17 @@ class Hashes(object): return self.check_against_chunks(read_chunks(file)) def check_against_path(self, path): + # type: (str) -> None with open(path, 'rb') as file: return self.check_against_file(file) def __nonzero__(self): + # type: () -> bool """Return whether I know any known-good hashes.""" return bool(self._allowed) def __bool__(self): + # type: () -> bool return self.__nonzero__() @@ -85,10 +104,12 @@ class MissingHashes(Hashes): """ def __init__(self): + # type: () -> None """Don't offer the ``hashes`` kwarg.""" # Pass our favorite hash in to generate a "gotten hash". With the # empty list, it will never match, so an error will always raise. super(MissingHashes, self).__init__(hashes={FAVORITE_HASH: []}) def _raise(self, gots): + # type: (Dict[str, _Hash]) -> NoReturn raise HashMissing(gots[FAVORITE_HASH].hexdigest()) diff --git a/pipenv/patched/notpip/_internal/utils/logging.py b/pipenv/patched/notpip/_internal/utils/logging.py index 576c4fa0..638c5ca9 100644 --- a/pipenv/patched/notpip/_internal/utils/logging.py +++ b/pipenv/patched/notpip/_internal/utils/logging.py @@ -1,9 +1,13 @@ from __future__ import absolute_import import contextlib +import errno import logging import logging.handlers import os +import sys + +from pipenv.patched.notpip._vendor.six import PY2 from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.misc import ensure_dir @@ -26,6 +30,48 @@ _log_state = threading.local() _log_state.indentation = 0 +class BrokenStdoutLoggingError(Exception): + """ + Raised if BrokenPipeError occurs for the stdout stream while logging. + """ + pass + + +# BrokenPipeError does not exist in Python 2 and, in addition, manifests +# differently in Windows and non-Windows. +if WINDOWS: + # In Windows, a broken pipe can show up as EINVAL rather than EPIPE: + # https://bugs.python.org/issue19612 + # https://bugs.python.org/issue30418 + if PY2: + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return (exc_class is IOError and + exc.errno in (errno.EINVAL, errno.EPIPE)) + else: + # In Windows, a broken pipe IOError became OSError in Python 3. + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return ((exc_class is BrokenPipeError) or # noqa: F821 + (exc_class is OSError and + exc.errno in (errno.EINVAL, errno.EPIPE))) +elif PY2: + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return (exc_class is IOError and exc.errno == errno.EPIPE) +else: + # Then we are in the non-Windows Python 3 case. + def _is_broken_pipe_error(exc_class, exc): + """ + Return whether an exception is a broken pipe error. + + Args: + exc_class: an exception class. + exc: an exception instance. + """ + return (exc_class is BrokenPipeError) # noqa: F821 + + @contextlib.contextmanager def indent_log(num=2): """ @@ -44,15 +90,28 @@ def get_indentation(): class IndentingFormatter(logging.Formatter): + def __init__(self, *args, **kwargs): + """ + A logging.Formatter obeying containing indent_log contexts. + + :param add_timestamp: A bool indicating output lines should be prefixed + with their record's timestamp. + """ + self.add_timestamp = kwargs.pop("add_timestamp", False) + super(IndentingFormatter, self).__init__(*args, **kwargs) def format(self, record): """ Calls the standard formatter, but will indent all of the log messages by our current indentation level. """ - formatted = logging.Formatter.format(self, record) + formatted = super(IndentingFormatter, self).format(record) + prefix = '' + if self.add_timestamp: + prefix = self.formatTime(record, "%Y-%m-%dT%H:%M:%S ") + prefix += " " * get_indentation() formatted = "".join([ - (" " * get_indentation()) + line + prefix + line for line in formatted.splitlines(True) ]) return formatted @@ -83,6 +142,16 @@ class ColorizedStreamHandler(logging.StreamHandler): if WINDOWS and colorama: self.stream = colorama.AnsiToWin32(self.stream) + def _using_stdout(self): + """ + Return whether the handler is using sys.stdout. + """ + if WINDOWS and colorama: + # Then self.stream is an AnsiToWin32 object. + return self.stream.wrapped is sys.stdout + + return self.stream is sys.stdout + def should_color(self): # Don't colorize things if we do not have colorama or if told not to if not colorama or self._no_color: @@ -115,6 +184,19 @@ class ColorizedStreamHandler(logging.StreamHandler): return msg + # The logging module says handleError() can be customized. + def handleError(self, record): + exc_class, exc = sys.exc_info()[:2] + # If a broken pipe occurred while calling write() or flush() on the + # stdout stream in logging's Handler.emit(), then raise our special + # exception so we can handle it in main() instead of logging the + # broken pipe error and continuing. + if (exc_class and self._using_stdout() and + _is_broken_pipe_error(exc_class, exc)): + raise BrokenStdoutLoggingError() + + return super(ColorizedStreamHandler, self).handleError(record) + class BetterRotatingFileHandler(logging.handlers.RotatingFileHandler): @@ -134,6 +216,8 @@ class MaxLevelFilter(logging.Filter): def setup_logging(verbosity, no_color, user_log_file): """Configures and sets up all of the logging + + Returns the requested logging level, as its integer value. """ # Determine the level to be logging at. @@ -148,6 +232,8 @@ def setup_logging(verbosity, no_color, user_log_file): else: level = "INFO" + level_number = getattr(logging, level) + # The "root" logger should match the "console" level *unless* we also need # to log to a user log file. include_user_log = user_log_file is not None @@ -186,6 +272,11 @@ def setup_logging(verbosity, no_color, user_log_file): "()": IndentingFormatter, "format": "%(message)s", }, + "indent_with_timestamp": { + "()": IndentingFormatter, + "format": "%(message)s", + "add_timestamp": True, + }, }, "handlers": { "console": { @@ -208,7 +299,7 @@ def setup_logging(verbosity, no_color, user_log_file): "class": handler_classes["file"], "filename": additional_log_file, "delay": True, - "formatter": "indent", + "formatter": "indent_with_timestamp", }, }, "root": { @@ -223,3 +314,5 @@ def setup_logging(verbosity, no_color, user_log_file): } }, }) + + return level_number diff --git a/pipenv/patched/notpip/_internal/utils/misc.py b/pipenv/patched/notpip/_internal/utils/misc.py index 45e5204c..a80153df 100644 --- a/pipenv/patched/notpip/_internal/utils/misc.py +++ b/pipenv/patched/notpip/_internal/utils/misc.py @@ -25,6 +25,7 @@ from pipenv.patched.notpip._vendor.retrying import retry # type: ignore from pipenv.patched.notpip._vendor.six import PY2 from pipenv.patched.notpip._vendor.six.moves import input from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse +from pipenv.patched.notpip._vendor.six.moves.urllib.parse import unquote as urllib_unquote from pipenv.patched.notpip._internal.exceptions import CommandError, InstallationError from pipenv.patched.notpip._internal.locations import ( @@ -34,12 +35,23 @@ from pipenv.patched.notpip._internal.locations import ( from pipenv.patched.notpip._internal.utils.compat import ( WINDOWS, console_to_str, expanduser, stdlib_pkgs, ) +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if PY2: from io import BytesIO as StringIO else: from io import StringIO +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Optional, Tuple, Iterable, List, Match, Union, Any, Mapping, Text, + AnyStr, Container + ) + from pipenv.patched.notpip._vendor.pkg_resources import Distribution # noqa: F401 + from pipenv.patched.notpip._internal.models.link import Link # noqa: F401 + from pipenv.patched.notpip._internal.utils.ui import SpinnerInterface # noqa: F401 + + __all__ = ['rmtree', 'display_path', 'backup_dir', 'ask', 'splitext', 'format_size', 'is_installable_dir', @@ -49,19 +61,21 @@ __all__ = ['rmtree', 'display_path', 'backup_dir', 'renames', 'get_prog', 'unzip_file', 'untar_file', 'unpack_file', 'call_subprocess', 'captured_stdout', 'ensure_dir', - 'ARCHIVE_EXTENSIONS', 'SUPPORTED_EXTENSIONS', + 'ARCHIVE_EXTENSIONS', 'SUPPORTED_EXTENSIONS', 'WHEEL_EXTENSION', 'get_installed_version', 'remove_auth_from_url'] logger = std_logging.getLogger(__name__) +WHEEL_EXTENSION = '.whl' BZ2_EXTENSIONS = ('.tar.bz2', '.tbz') XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', '.tar.lz', '.tar.lzma') -ZIP_EXTENSIONS = ('.zip', '.whl') +ZIP_EXTENSIONS = ('.zip', WHEEL_EXTENSION) TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar') ARCHIVE_EXTENSIONS = ( ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS) SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS + try: import bz2 # noqa SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS @@ -76,14 +90,8 @@ except ImportError: logger.debug('lzma module is not available') -def import_or_raise(pkg_or_module_string, ExceptionType, *args, **kwargs): - try: - return __import__(pkg_or_module_string) - except ImportError: - raise ExceptionType(*args, **kwargs) - - def ensure_dir(path): + # type: (AnyStr) -> None """os.path.makedirs without EEXIST.""" try: os.makedirs(path) @@ -93,6 +101,7 @@ def ensure_dir(path): def get_prog(): + # type: () -> str try: prog = os.path.basename(sys.argv[0]) if prog in ('__main__.py', '-c'): @@ -107,6 +116,7 @@ def get_prog(): # Retry every half second for up to 3 seconds @retry(stop_max_delay=3000, wait_fixed=500) def rmtree(dir, ignore_errors=False): + # type: (str, bool) -> None shutil.rmtree(dir, ignore_errors=ignore_errors, onerror=rmtree_errorhandler) @@ -127,6 +137,7 @@ def rmtree_errorhandler(func, path, exc_info): def display_path(path): + # type: (Union[str, Text]) -> str """Gives the display value for a given path, making it relative to cwd if possible.""" path = os.path.normcase(os.path.abspath(path)) @@ -139,6 +150,7 @@ def display_path(path): def backup_dir(dir, ext='.bak'): + # type: (str, str) -> str """Figure out the name of a directory to back up the given dir to (adding .bak, .bak2, etc)""" n = 1 @@ -150,6 +162,7 @@ def backup_dir(dir, ext='.bak'): def ask_path_exists(message, options): + # type: (str, Iterable[str]) -> str for action in os.environ.get('PIP_EXISTS_ACTION', '').split(): if action in options: return action @@ -157,6 +170,7 @@ def ask_path_exists(message, options): def ask(message, options): + # type: (str, Iterable[str]) -> str """Ask the message interactively, with the given possible responses""" while 1: if os.environ.get('PIP_NO_INPUT'): @@ -176,6 +190,7 @@ def ask(message, options): def format_size(bytes): + # type: (float) -> str if bytes > 1000 * 1000: return '%.1fMB' % (bytes / 1000.0 / 1000) elif bytes > 10 * 1000: @@ -187,6 +202,7 @@ def format_size(bytes): def is_installable_dir(path): + # type: (str) -> bool """Is path is a directory containing setup.py or pyproject.toml? """ if not os.path.isdir(path): @@ -201,6 +217,7 @@ def is_installable_dir(path): def is_svn_page(html): + # type: (Union[str, Text]) -> Optional[Match[Union[str, Text]]] """ Returns true if the page appears to be the index page of an svn repository """ @@ -209,6 +226,7 @@ def is_svn_page(html): def file_contents(filename): + # type: (str) -> Text with open(filename, 'rb') as fp: return fp.read().decode('utf-8') @@ -223,6 +241,7 @@ def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): def split_leading_dir(path): + # type: (Union[str, Text]) -> List[Union[str, Text]] path = path.lstrip('/').lstrip('\\') if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) or '\\' not in path): @@ -230,10 +249,11 @@ def split_leading_dir(path): elif '\\' in path: return path.split('\\', 1) else: - return path, '' + return [path, ''] def has_leading_dir(paths): + # type: (Iterable[Union[str, Text]]) -> bool """Returns true if all the paths have the same leading path name (i.e., everything is in one subdirectory in an archive)""" common_prefix = None @@ -249,6 +269,7 @@ def has_leading_dir(paths): def normalize_path(path, resolve_symlinks=True): + # type: (str, bool) -> str """ Convert a path to its canonical, case-normalized, absolute version. @@ -262,6 +283,7 @@ def normalize_path(path, resolve_symlinks=True): def splitext(path): + # type: (str) -> Tuple[str, str] """Like os.path.splitext, but take off .tar too""" base, ext = posixpath.splitext(path) if base.lower().endswith('.tar'): @@ -271,6 +293,7 @@ def splitext(path): def renames(old, new): + # type: (str, str) -> None """Like os.renames(), but handles renaming across devices.""" # Implementation borrowed from os.renames(). head, tail = os.path.split(new) @@ -288,6 +311,7 @@ def renames(old, new): def is_local(path): + # type: (str) -> bool """ Return True if path is within sys.prefix, if we're running in a virtualenv. @@ -300,6 +324,7 @@ def is_local(path): def dist_is_local(dist): + # type: (Distribution) -> bool """ Return True if given Distribution object is installed locally (i.e. within current virtualenv). @@ -311,6 +336,7 @@ def dist_is_local(dist): def dist_in_usersite(dist): + # type: (Distribution) -> bool """ Return True if given Distribution is installed in user site. """ @@ -319,6 +345,7 @@ def dist_in_usersite(dist): def dist_in_site_packages(dist): + # type: (Distribution) -> bool """ Return True if given Distribution is installed in sysconfig.get_python_lib(). @@ -329,7 +356,10 @@ def dist_in_site_packages(dist): def dist_is_editable(dist): - """Is distribution an editable install?""" + # type: (Distribution) -> bool + """ + Return True if given Distribution is an editable install. + """ for path_item in sys.path: egg_link = os.path.join(path_item, dist.project_name + '.egg-link') if os.path.isfile(egg_link): @@ -342,6 +372,7 @@ def get_installed_distributions(local_only=True, include_editables=True, editables_only=False, user_only=False): + # type: (bool, Container[str], bool, bool, bool) -> List[Distribution] """ Return a list of installed Distribution objects. @@ -385,7 +416,8 @@ def get_installed_distributions(local_only=True, def user_test(d): return True - return [d for d in pkg_resources.working_set + # because of pkg_resources vendoring, mypy cannot find stub in typeshed + return [d for d in pkg_resources.working_set # type: ignore if local_test(d) and d.key not in skip and editable_test(d) and @@ -395,6 +427,7 @@ def get_installed_distributions(local_only=True, def egg_link_path(dist): + # type: (Distribution) -> Optional[str] """ Return the path for the .egg-link file if it exists, otherwise, None. @@ -429,9 +462,11 @@ def egg_link_path(dist): egglink = os.path.join(site, dist.project_name) + '.egg-link' if os.path.isfile(egglink): return egglink + return None def dist_location(dist): + # type: (Distribution) -> str """ Get the site-packages location of this distribution. Generally this is dist.location, except in the case of develop-installed @@ -453,6 +488,7 @@ def current_umask(): def unzip_file(filename, location, flatten=True): + # type: (str, str, bool) -> None """ Unzip the file (with path `filename`) to the destination `location`. All files are written based on system defaults and umask (i.e. permissions are @@ -468,7 +504,6 @@ def unzip_file(filename, location, flatten=True): leading = has_leading_dir(zip.namelist()) and flatten for info in zip.infolist(): name = info.filename - data = zip.read(name) fn = name if leading: fn = split_leading_dir(name)[1] @@ -479,9 +514,12 @@ def unzip_file(filename, location, flatten=True): ensure_dir(fn) else: ensure_dir(dir) - fp = open(fn, 'wb') + # Don't use read() to avoid allocating an arbitrarily large + # chunk of memory for the file's content + fp = zip.open(name) try: - fp.write(data) + with open(fn, 'wb') as destfp: + shutil.copyfileobj(fp, destfp) finally: fp.close() mode = info.external_attr >> 16 @@ -496,6 +534,7 @@ def unzip_file(filename, location, flatten=True): def untar_file(filename, location): + # type: (str, str) -> None """ Untar the file (with path `filename`) to the destination `location`. All files are written based on system defaults and umask (i.e. permissions @@ -520,23 +559,21 @@ def untar_file(filename, location): mode = 'r:*' tar = tarfile.open(filename, mode) try: - # note: python<=2.5 doesn't seem to know about pax headers, filter them leading = has_leading_dir([ member.name for member in tar.getmembers() - if member.name != 'pax_global_header' ]) for member in tar.getmembers(): fn = member.name - if fn == 'pax_global_header': - continue if leading: - fn = split_leading_dir(fn)[1] + # https://github.com/python/mypy/issues/1174 + fn = split_leading_dir(fn)[1] # type: ignore path = os.path.join(location, fn) if member.isdir(): ensure_dir(path) elif member.issym(): try: - tar._extract_member(member, path) + # https://github.com/python/typeshed/issues/2673 + tar._extract_member(member, path) # type: ignore except Exception as exc: # Some corrupt tar files seem to produce this # (specifically bad symlinks) @@ -561,7 +598,8 @@ def untar_file(filename, location): shutil.copyfileobj(fp, destfp) fp.close() # Update the timestamp (useful for cython compiled files) - tar.utime(member, path) + # https://github.com/python/typeshed/issues/2673 + tar.utime(member, path) # type: ignore # member have any execute permissions for user/group/world? if member.mode & 0o111: # make dest file have execute for user/group/world @@ -571,7 +609,13 @@ def untar_file(filename, location): tar.close() -def unpack_file(filename, location, content_type, link): +def unpack_file( + filename, # type: str + location, # type: str + content_type, # type: Optional[str] + link # type: Optional[Link] +): + # type: (...) -> None filename = os.path.realpath(filename) if (content_type == 'application/zip' or filename.lower().endswith(ZIP_EXTENSIONS) or @@ -604,15 +648,27 @@ def unpack_file(filename, location, content_type, link): ) -def call_subprocess(cmd, show_stdout=True, cwd=None, - on_returncode='raise', - command_desc=None, - extra_environ=None, unset_environ=None, spinner=None): +def call_subprocess( + cmd, # type: List[str] + show_stdout=True, # type: bool + cwd=None, # type: Optional[str] + on_returncode='raise', # type: str + extra_ok_returncodes=None, # type: Optional[Iterable[int]] + command_desc=None, # type: Optional[str] + extra_environ=None, # type: Optional[Mapping[str, Any]] + unset_environ=None, # type: Optional[Iterable[str]] + spinner=None # type: Optional[SpinnerInterface] +): + # type: (...) -> Optional[Text] """ Args: + extra_ok_returncodes: an iterable of integer return codes that are + acceptable, in addition to 0. Defaults to None, which means []. unset_environ: an iterable of environment variable names to unset prior to calling subprocess.Popen(). """ + if extra_ok_returncodes is None: + extra_ok_returncodes = [] if unset_environ is None: unset_environ = [] # This function's handling of subprocess output is confusing and I @@ -689,7 +745,7 @@ def call_subprocess(cmd, show_stdout=True, cwd=None, spinner.finish("error") else: spinner.finish("done") - if proc.returncode: + if proc.returncode and proc.returncode not in extra_ok_returncodes: if on_returncode == 'raise': if (logger.getEffectiveLevel() > std_logging.DEBUG and not show_stdout): @@ -715,9 +771,11 @@ def call_subprocess(cmd, show_stdout=True, cwd=None, repr(on_returncode)) if not show_stdout: return ''.join(all_output) + return None def read_text_file(filename): + # type: (str) -> str """Return the contents of *filename*. Try to decode the file contents with utf-8, the preferred system encoding @@ -732,12 +790,13 @@ def read_text_file(filename): encodings = ['utf-8', locale.getpreferredencoding(False), 'latin1'] for enc in encodings: try: - data = data.decode(enc) + # https://github.com/python/mypy/issues/1174 + data = data.decode(enc) # type: ignore except UnicodeDecodeError: continue break - assert type(data) != bytes # Latin1 should have worked. + assert not isinstance(data, bytes) # Latin1 should have worked. return data @@ -805,6 +864,13 @@ def captured_stdout(): return captured_output('stdout') +def captured_stderr(): + """ + See captured_stdout(). + """ + return captured_output('stderr') + + class cached_property(object): """A property that is only computed once per instance and then replaces itself with an ordinary attribute. Deleting the attribute resets the @@ -856,13 +922,15 @@ def enum(*sequential, **named): return type('Enum', (), enums) -def make_vcs_requirement_url(repo_url, rev, egg_project_name, subdir=None): +def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): """ Return the URL for a VCS requirement. Args: repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + project_name: the (unescaped) project name. """ + egg_project_name = pkg_resources.to_filename(project_name) req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) if subdir: req += '&subdirectory={}'.format(subdir) @@ -887,22 +955,36 @@ def split_auth_from_netloc(netloc): # Split from the left because that's how urllib.parse.urlsplit() # behaves if more than one : is present (which again can be checked # using the password attribute of the return value) - user_pass = tuple(auth.split(':', 1)) + user_pass = auth.split(':', 1) else: user_pass = auth, None + user_pass = tuple( + None if x is None else urllib_unquote(x) for x in user_pass + ) + return netloc, user_pass -def remove_auth_from_url(url): - # Return a copy of url with 'username:password@' removed. - # username/pass params are passed to subversion through flags - # and are not recognized in the url. +def redact_netloc(netloc): + # type: (str) -> str + """ + Replace the password in a netloc with "****", if it exists. - # parsed url + For example, "user:pass@example.com" returns "user:****@example.com". + """ + netloc, (user, password) = split_auth_from_netloc(netloc) + if user is None: + return netloc + password = '' if password is None else ':****' + return '{user}{password}@{netloc}'.format(user=urllib_parse.quote(user), + password=password, + netloc=netloc) + + +def _transform_url(url, transform_netloc): purl = urllib_parse.urlsplit(url) - netloc, user_pass = split_auth_from_netloc(purl.netloc) - + netloc = transform_netloc(purl.netloc) # stripped url url_pieces = ( purl.scheme, netloc, purl.path, purl.query, purl.fragment @@ -911,6 +993,24 @@ def remove_auth_from_url(url): return surl +def _get_netloc(netloc): + return split_auth_from_netloc(netloc)[0] + + +def remove_auth_from_url(url): + # type: (str) -> str + # Return a copy of url with 'username:password@' removed. + # username/pass params are passed to subversion through flags + # and are not recognized in the url. + return _transform_url(url, _get_netloc) + + +def redact_password_from_url(url): + # type: (str) -> str + """Replace the password in a given url with ****.""" + return _transform_url(url, redact_netloc) + + def protect_pip_from_modification_on_windows(modifying_pip): """Protection of pip.exe from modification on Windows diff --git a/pipenv/patched/notpip/_internal/utils/outdated.py b/pipenv/patched/notpip/_internal/utils/outdated.py index f8b1fe04..83dc58cc 100644 --- a/pipenv/patched/notpip/_internal/utils/outdated.py +++ b/pipenv/patched/notpip/_internal/utils/outdated.py @@ -13,6 +13,13 @@ from pipenv.patched.notpip._internal.index import PackageFinder from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner from pipenv.patched.notpip._internal.utils.misc import ensure_dir, get_installed_version +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + import optparse # noqa: F401 + from typing import Any, Dict # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ" @@ -22,7 +29,8 @@ logger = logging.getLogger(__name__) class SelfCheckState(object): def __init__(self, cache_dir): - self.state = {} + # type: (str) -> None + self.state = {} # type: Dict[str, Any] self.statefile_path = None # Try to load the existing state @@ -37,6 +45,7 @@ class SelfCheckState(object): pass def save(self, pypi_version, current_time): + # type: (str, datetime.datetime) -> None # If we do not have a path to cache in, don't bother saving. if not self.statefile_path: return @@ -68,6 +77,7 @@ class SelfCheckState(object): def was_installed_by_pip(pkg): + # type: (str) -> bool """Checks whether pkg was installed by pip This is used not to display the upgrade message when pip is in fact @@ -82,6 +92,7 @@ def was_installed_by_pip(pkg): def pip_version_check(session, options): + # type: (PipSession, optparse.Values) -> None """Check for an update for pip. Limit the frequency of checks to once per week. State is stored either in @@ -116,7 +127,6 @@ def pip_version_check(session, options): index_urls=[options.index_url] + options.extra_index_urls, allow_all_prereleases=False, # Explicitly set to False trusted_hosts=options.trusted_hosts, - process_dependency_links=options.process_dependency_links, session=session, ) all_candidates = finder.find_all_candidates("pip") diff --git a/pipenv/patched/notpip/_internal/utils/packaging.py b/pipenv/patched/notpip/_internal/utils/packaging.py index d1e8ecaa..dc944529 100644 --- a/pipenv/patched/notpip/_internal/utils/packaging.py +++ b/pipenv/patched/notpip/_internal/utils/packaging.py @@ -2,18 +2,26 @@ from __future__ import absolute_import import logging import sys -from email.parser import FeedParser # type: ignore +from email.parser import FeedParser from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.packaging import specifiers, version from pipenv.patched.notpip._internal import exceptions from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional # noqa: F401 + from email.message import Message # noqa: F401 + from pipenv.patched.notpip._vendor.pkg_resources import Distribution # noqa: F401 + logger = logging.getLogger(__name__) def check_requires_python(requires_python): + # type: (Optional[str]) -> bool """ Check if the python version in use match the `requires_python` specifier. @@ -34,6 +42,7 @@ def check_requires_python(requires_python): def get_metadata(dist): + # type: (Distribution) -> Message if (isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata('METADATA')): metadata = dist.get_metadata('METADATA') @@ -48,7 +57,7 @@ def get_metadata(dist): return feed_parser.close() -def check_dist_requires_python(dist, absorb=True): +def check_dist_requires_python(dist, absorb=False): pkg_info_dict = get_metadata(dist) requires_python = pkg_info_dict.get('Requires-Python') if absorb: @@ -70,6 +79,7 @@ def check_dist_requires_python(dist, absorb=True): def get_installer(dist): + # type: (Distribution) -> str if dist.has_metadata('INSTALLER'): for line in dist.get_metadata_lines('INSTALLER'): if line.strip(): diff --git a/pipenv/patched/notpip/_internal/utils/temp_dir.py b/pipenv/patched/notpip/_internal/utils/temp_dir.py index 893dc975..d8911bc9 100644 --- a/pipenv/patched/notpip/_internal/utils/temp_dir.py +++ b/pipenv/patched/notpip/_internal/utils/temp_dir.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +import errno +import itertools import logging import os.path import tempfile @@ -57,7 +59,7 @@ class TempDirectory(object): self, self._cleanup, self.path, - warn_message=None + warn_message = None ) else: self._finalizer = None @@ -74,7 +76,7 @@ class TempDirectory(object): self.cleanup() def create(self): - """Create a temporary directory and store it's path in self.path + """Create a temporary directory and store its path in self.path """ if self.path is not None: logger.debug( @@ -112,3 +114,74 @@ class TempDirectory(object): pass else: self.path = None + + +class AdjacentTempDirectory(TempDirectory): + """Helper class that creates a temporary directory adjacent to a real one. + + Attributes: + original + The original directory to create a temp directory for. + path + After calling create() or entering, contains the full + path to the temporary directory. + delete + Whether the directory should be deleted when exiting + (when used as a contextmanager) + + """ + # The characters that may be used to name the temp directory + # We always prepend a ~ and then rotate through these until + # a usable name is found. + # pkg_resources raises a different error for .dist-info folder + # with leading '-' and invalid metadata + LEADING_CHARS = "-~.=%0123456789" + + def __init__(self, original, delete=None): + super(AdjacentTempDirectory, self).__init__(delete=delete) + self.original = original.rstrip('/\\') + + @classmethod + def _generate_names(cls, name): + """Generates a series of temporary names. + + The algorithm replaces the leading characters in the name + with ones that are valid filesystem characters, but are not + valid package names (for both Python and pip definitions of + package). + """ + for i in range(1, len(name)): + for candidate in itertools.combinations_with_replacement( + cls.LEADING_CHARS, i - 1): + new_name = '~' + ''.join(candidate) + name[i:] + if new_name != name: + yield new_name + + # If we make it this far, we will have to make a longer name + for i in range(len(cls.LEADING_CHARS)): + for candidate in itertools.combinations_with_replacement( + cls.LEADING_CHARS, i): + new_name = '~' + ''.join(candidate) + name + if new_name != name: + yield new_name + + def create(self): + root, name = os.path.split(self.original) + for candidate in self._generate_names(name): + path = os.path.join(root, candidate) + try: + os.mkdir(path) + except OSError as ex: + # Continue if the name exists already + if ex.errno != errno.EEXIST: + raise + else: + self.path = os.path.realpath(path) + break + + if not self.path: + # Final fallback on the default behavior. + self.path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) + ) + logger.debug("Created temporary directory: {}".format(self.path)) diff --git a/pipenv/patched/notpip/_internal/utils/ui.py b/pipenv/patched/notpip/_internal/utils/ui.py index 6eebd17d..18119e0e 100644 --- a/pipenv/patched/notpip/_internal/utils/ui.py +++ b/pipenv/patched/notpip/_internal/utils/ui.py @@ -21,7 +21,7 @@ from pipenv.patched.notpip._internal.utils.misc import format_size from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any # noqa: F401 + from typing import Any, Iterator, IO # noqa: F401 try: from pipenv.patched.notpip._vendor import colorama @@ -203,7 +203,7 @@ class BaseDownloadProgressBar(WindowsMixin, InterruptibleMixin, class DefaultDownloadProgressBar(BaseDownloadProgressBar, - _BaseBar): # type: ignore + _BaseBar): pass @@ -292,6 +292,7 @@ def DownloadProgressProvider(progress_bar, max=None): @contextlib.contextmanager def hidden_cursor(file): + # type: (IO) -> Iterator[None] # The Windows terminal does not support the hide/show cursor ANSI codes, # even via colorama. So don't even try. if WINDOWS: @@ -311,19 +312,32 @@ def hidden_cursor(file): class RateLimiter(object): def __init__(self, min_update_interval_seconds): + # type: (float) -> None self._min_update_interval_seconds = min_update_interval_seconds - self._last_update = 0 + self._last_update = 0 # type: float def ready(self): + # type: () -> bool now = time.time() delta = now - self._last_update return delta >= self._min_update_interval_seconds def reset(self): + # type: () -> None self._last_update = time.time() -class InteractiveSpinner(object): +class SpinnerInterface(object): + def spin(self): + # type: () -> None + raise NotImplementedError() + + def finish(self, final_status): + # type: (str) -> None + raise NotImplementedError() + + +class InteractiveSpinner(SpinnerInterface): def __init__(self, message, file=None, spin_chars="-\\|/", # Empirically, 8 updates/second looks nice min_update_interval_seconds=0.125): @@ -352,6 +366,7 @@ class InteractiveSpinner(object): self._rate_limiter.reset() def spin(self): + # type: () -> None if self._finished: return if not self._rate_limiter.ready(): @@ -359,6 +374,7 @@ class InteractiveSpinner(object): self._write(next(self._spin_cycle)) def finish(self, final_status): + # type: (str) -> None if self._finished: return self._write(final_status) @@ -371,8 +387,9 @@ class InteractiveSpinner(object): # We still print updates occasionally (once every 60 seconds by default) to # act as a keep-alive for systems like Travis-CI that take lack-of-output as # an indication that a task has frozen. -class NonInteractiveSpinner(object): +class NonInteractiveSpinner(SpinnerInterface): def __init__(self, message, min_update_interval_seconds=60): + # type: (str, float) -> None self._message = message self._finished = False self._rate_limiter = RateLimiter(min_update_interval_seconds) @@ -384,6 +401,7 @@ class NonInteractiveSpinner(object): logger.info("%s: %s", self._message, status) def spin(self): + # type: () -> None if self._finished: return if not self._rate_limiter.ready(): @@ -391,6 +409,7 @@ class NonInteractiveSpinner(object): self._update("still running...") def finish(self, final_status): + # type: (str) -> None if self._finished: return self._update("finished with status '%s'" % (final_status,)) @@ -399,13 +418,14 @@ class NonInteractiveSpinner(object): @contextlib.contextmanager def open_spinner(message): + # type: (str) -> Iterator[SpinnerInterface] # Interactive spinner goes directly to sys.stdout rather than being routed # through the logging system, but it acts like it has level INFO, # i.e. it's only displayed if we're at level INFO or better. # Non-interactive spinner goes through the logging system, so it is always # in sync with logging configuration. if sys.stdout.isatty() and logger.getEffectiveLevel() <= logging.INFO: - spinner = InteractiveSpinner(message) + spinner = InteractiveSpinner(message) # type: SpinnerInterface else: spinner = NonInteractiveSpinner(message) try: diff --git a/pipenv/patched/notpip/_internal/vcs/__init__.py b/pipenv/patched/notpip/_internal/vcs/__init__.py index 5aeac633..06d36148 100644 --- a/pipenv/patched/notpip/_internal/vcs/__init__.py +++ b/pipenv/patched/notpip/_internal/vcs/__init__.py @@ -16,15 +16,23 @@ from pipenv.patched.notpip._internal.utils.misc import ( from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Dict, Optional, Tuple # noqa: F401 - from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 + from typing import ( # noqa: F401 + Any, Dict, Iterable, List, Mapping, Optional, Text, Tuple, Type + ) + from pipenv.patched.notpip._internal.utils.ui import SpinnerInterface # noqa: F401 -__all__ = ['vcs', 'get_src_requirement'] + AuthInfo = Tuple[Optional[str], Optional[str]] + +__all__ = ['vcs'] logger = logging.getLogger(__name__) +class RemoteNotFoundError(Exception): + pass + + class RevOptions(object): """ @@ -35,6 +43,7 @@ class RevOptions(object): """ def __init__(self, vcs, rev=None, extra_args=None): + # type: (VersionControl, Optional[str], Optional[List[str]]) -> None """ Args: vcs: a VersionControl object. @@ -53,16 +62,18 @@ class RevOptions(object): @property def arg_rev(self): + # type: () -> Optional[str] if self.rev is None: return self.vcs.default_arg_rev return self.rev def to_args(self): + # type: () -> List[str] """ Return the VCS-specific command arguments. """ - args = [] + args = [] # type: List[str] rev = self.arg_rev if rev is not None: args += self.vcs.get_base_rev_args(rev) @@ -71,12 +82,14 @@ class RevOptions(object): return args def to_display(self): + # type: () -> str if not self.rev: return '' return ' (to revision {})'.format(self.rev) def make_new(self, rev): + # type: (str) -> RevOptions """ Make a copy of the current instance, but with a new rev. @@ -87,10 +100,11 @@ class RevOptions(object): class VcsSupport(object): - _registry = {} # type: Dict[str, Command] + _registry = {} # type: Dict[str, Type[VersionControl]] schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] def __init__(self): + # type: () -> None # Register more schemes with urlparse for various version control # systems urllib_parse.uses_netloc.extend(self.schemes) @@ -104,20 +118,24 @@ class VcsSupport(object): @property def backends(self): + # type: () -> List[Type[VersionControl]] return list(self._registry.values()) @property def dirnames(self): + # type: () -> List[str] return [backend.dirname for backend in self.backends] @property def all_schemes(self): - schemes = [] + # type: () -> List[str] + schemes = [] # type: List[str] for backend in self.backends: schemes.extend(backend.schemes) return schemes def register(self, cls): + # type: (Type[VersionControl]) -> None if not hasattr(cls, 'name'): logger.warning('Cannot register VCS %s', cls.__name__) return @@ -126,6 +144,7 @@ class VcsSupport(object): logger.debug('Registered VCS backend: %s', cls.name) def unregister(self, cls=None, name=None): + # type: (Optional[Type[VersionControl]], Optional[str]) -> None if name in self._registry: del self._registry[name] elif cls in self._registry.values(): @@ -133,27 +152,24 @@ class VcsSupport(object): else: logger.warning('Cannot unregister because no class or name given') - def get_backend_name(self, location): + def get_backend_type(self, location): + # type: (str) -> Optional[Type[VersionControl]] """ - Return the name of the version control backend if found at given - location, e.g. vcs.get_backend_name('/path/to/vcs/checkout') + Return the type of the version control backend if found at given + location, e.g. vcs.get_backend_type('/path/to/vcs/checkout') """ for vc_type in self._registry.values(): if vc_type.controls_location(location): logger.debug('Determine that %s uses VCS: %s', location, vc_type.name) - return vc_type.name + return vc_type return None def get_backend(self, name): + # type: (str) -> Optional[Type[VersionControl]] name = name.lower() if name in self._registry: return self._registry[name] - - def get_backend_from_location(self, location): - vc_type = self.get_backend_name(location) - if vc_type: - return self.get_backend(vc_type) return None @@ -163,6 +179,7 @@ vcs = VcsSupport() class VersionControl(object): name = '' dirname = '' + repo_name = '' # List of supported schemes for this Version Control schemes = () # type: Tuple[str, ...] # Iterable of environment variable names to pass to call_subprocess(). @@ -183,6 +200,7 @@ class VersionControl(object): raise NotImplementedError def make_rev_options(self, rev=None, extra_args=None): + # type: (Optional[str], Optional[List[str]]) -> RevOptions """ Return a RevOptions object. @@ -192,13 +210,15 @@ class VersionControl(object): """ return RevOptions(self, rev, extra_args=extra_args) - def _is_local_repository(self, repo): + @classmethod + def _is_local_repository(cls, repo): + # type: (str) -> bool """ posix absolute paths start with os.path.sep, win32 ones start with drive (like c:\\folder) """ drive, tail = os.path.splitdrive(repo) - return repo.startswith(os.path.sep) or drive + return repo.startswith(os.path.sep) or bool(drive) def export(self, location): """ @@ -226,6 +246,7 @@ class VersionControl(object): return netloc, (None, None) def get_url_rev_and_auth(self, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] """ Parse the repository URL to use, and return the URL, revision, and auth info to use. @@ -255,6 +276,7 @@ class VersionControl(object): return [] def get_url_rev_options(self, url): + # type: (str) -> Tuple[str, RevOptions] """ Return the URL and RevOptions object to use in obtain() and in some cases export(), as a tuple (url, rev_options). @@ -267,6 +289,7 @@ class VersionControl(object): return url, rev_options def normalize_url(self, url): + # type: (str) -> str """ Normalize a URL for comparison by unquoting it and removing any trailing slash. @@ -274,6 +297,7 @@ class VersionControl(object): return urllib_parse.unquote(url).rstrip('/') def compare_urls(self, url1, url2): + # type: (str, str) -> bool """ Compare two repo URLs for identity, ignoring incidental differences. """ @@ -319,6 +343,7 @@ class VersionControl(object): raise NotImplementedError def obtain(self, dest): + # type: (str) -> None """ Install or update in editable mode the package represented by this VersionControl object. @@ -334,7 +359,7 @@ class VersionControl(object): rev_display = rev_options.to_display() if self.is_repository_directory(dest): - existing_url = self.get_url(dest) + existing_url = self.get_remote_url(dest) if self.compare_urls(existing_url, url): logger.debug( '%s in %s exists, and has correct URL (%s)', @@ -370,7 +395,9 @@ class VersionControl(object): self.name, self.repo_name, ) - prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) + # https://github.com/python/mypy/issues/1174 + prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore + ('i', 'w', 'b')) logger.warning( 'The plan is to install the %s repository %s', @@ -409,6 +436,7 @@ class VersionControl(object): self.switch(dest, url, rev_options) def unpack(self, location): + # type: (str) -> None """ Clean up current location and download the url repository (and vcs infos) into location @@ -417,7 +445,8 @@ class VersionControl(object): rmtree(location) self.obtain(location) - def get_src_requirement(self, dist, location): + @classmethod + def get_src_requirement(cls, location, project_name): """ Return a string representing the requirement needed to redownload the files currently present in location, something @@ -426,33 +455,49 @@ class VersionControl(object): """ raise NotImplementedError - def get_url(self, location): + @classmethod + def get_remote_url(cls, location): """ Return the url used at location + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. """ raise NotImplementedError - def get_revision(self, location): + @classmethod + def get_revision(cls, location): """ Return the current commit id of the files at the given location. """ raise NotImplementedError - def run_command(self, cmd, show_stdout=True, cwd=None, - on_returncode='raise', - command_desc=None, - extra_environ=None, spinner=None): + @classmethod + def run_command( + cls, + cmd, # type: List[str] + show_stdout=True, # type: bool + cwd=None, # type: Optional[str] + on_returncode='raise', # type: str + extra_ok_returncodes=None, # type: Optional[Iterable[int]] + command_desc=None, # type: Optional[str] + extra_environ=None, # type: Optional[Mapping[str, Any]] + spinner=None # type: Optional[SpinnerInterface] + ): + # type: (...) -> Optional[Text] """ Run a VCS subcommand This is simply a wrapper around call_subprocess that adds the VCS command name, and checks that the VCS is available """ - cmd = [self.name] + cmd + cmd = [cls.name] + cmd try: return call_subprocess(cmd, show_stdout, cwd, - on_returncode, - command_desc, extra_environ, - unset_environ=self.unset_environ, + on_returncode=on_returncode, + extra_ok_returncodes=extra_ok_returncodes, + command_desc=command_desc, + extra_environ=extra_environ, + unset_environ=cls.unset_environ, spinner=spinner) except OSError as e: # errno.ENOENT = no such file or directory @@ -461,12 +506,13 @@ class VersionControl(object): raise BadCommand( 'Cannot find command %r - do you have ' '%r installed and in your ' - 'PATH?' % (self.name, self.name)) + 'PATH?' % (cls.name, cls.name)) else: raise # re-raise exception if a different error occurred @classmethod def is_repository_directory(cls, path): + # type: (str) -> bool """ Return whether a directory path is a repository directory. """ @@ -476,6 +522,7 @@ class VersionControl(object): @classmethod def controls_location(cls, location): + # type: (str) -> bool """ Check if a location is controlled by the vcs. It is meant to be overridden to implement smarter detection @@ -485,25 +532,3 @@ class VersionControl(object): the Git override checks that Git is actually available. """ return cls.is_repository_directory(location) - - -def get_src_requirement(dist, location): - version_control = vcs.get_backend_from_location(location) - if version_control: - try: - return version_control().get_src_requirement(dist, - location) - except BadCommand: - logger.warning( - 'cannot determine version of editable source in %s ' - '(%s command not found in path)', - location, - version_control.name, - ) - return dist.as_requirement() - logger.warning( - 'cannot determine version of editable source in %s (is not SVN ' - 'checkout, Git clone, Mercurial clone or Bazaar branch)', - location, - ) - return dist.as_requirement() diff --git a/pipenv/patched/notpip/_internal/vcs/bazaar.py b/pipenv/patched/notpip/_internal/vcs/bazaar.py index 890448ed..256eb9e2 100644 --- a/pipenv/patched/notpip/_internal/vcs/bazaar.py +++ b/pipenv/patched/notpip/_internal/vcs/bazaar.py @@ -75,34 +75,36 @@ class Bazaar(VersionControl): url = 'bzr+' + url return url, rev, user_pass - def get_url(self, location): - urls = self.run_command(['info'], show_stdout=False, cwd=location) + @classmethod + def get_remote_url(cls, location): + urls = cls.run_command(['info'], show_stdout=False, cwd=location) for line in urls.splitlines(): line = line.strip() for x in ('checkout of branch: ', 'parent branch: '): if line.startswith(x): repo = line.split(x)[1] - if self._is_local_repository(repo): + if cls._is_local_repository(repo): return path_to_url(repo) return repo return None - def get_revision(self, location): - revision = self.run_command( + @classmethod + def get_revision(cls, location): + revision = cls.run_command( ['revno'], show_stdout=False, cwd=location, ) return revision.splitlines()[-1] - def get_src_requirement(self, dist, location): - repo = self.get_url(location) + @classmethod + def get_src_requirement(cls, location, project_name): + repo = cls.get_remote_url(location) if not repo: return None if not repo.lower().startswith('bzr:'): repo = 'bzr+' + repo - current_rev = self.get_revision(location) - egg_project_name = dist.egg_name().split('-', 1)[0] - return make_vcs_requirement_url(repo, current_rev, egg_project_name) + current_rev = cls.get_revision(location) + return make_vcs_requirement_url(repo, current_rev, project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/git.py b/pipenv/patched/notpip/_internal/vcs/git.py index 3db56144..310eb9fb 100644 --- a/pipenv/patched/notpip/_internal/vcs/git.py +++ b/pipenv/patched/notpip/_internal/vcs/git.py @@ -10,9 +10,11 @@ from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_req from pipenv.patched.notpip._internal.exceptions import BadCommand from pipenv.patched.notpip._internal.utils.compat import samefile -from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url +from pipenv.patched.notpip._internal.utils.misc import ( + display_path, make_vcs_requirement_url, redact_password_from_url, +) from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory -from pipenv.patched.notpip._internal.vcs import VersionControl, vcs +from pipenv.patched.notpip._internal.vcs import RemoteNotFoundError, VersionControl, vcs urlsplit = urllib_parse.urlsplit urlunsplit = urllib_parse.urlunsplit @@ -77,19 +79,25 @@ class Git(VersionControl): version = '.'.join(version.split('.')[:3]) return parse_version(version) - def get_branch(self, location): + def get_current_branch(self, location): """ Return the current branch, or None if HEAD isn't at a branch (e.g. detached HEAD). """ - args = ['rev-parse', '--abbrev-ref', 'HEAD'] - output = self.run_command(args, show_stdout=False, cwd=location) - branch = output.strip() + # git-symbolic-ref exits with empty stdout if "HEAD" is a detached + # HEAD rather than a symbolic ref. In addition, the -q causes the + # command to exit with status code 1 instead of 128 in this case + # and to suppress the message to stderr. + args = ['symbolic-ref', '-q', 'HEAD'] + output = self.run_command( + args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, + ) + ref = output.strip() - if branch == 'HEAD': - return None + if ref.startswith('refs/heads/'): + return ref[len('refs/heads/'):] - return branch + return None def export(self, location): """Export the Git repository at the url to the destination location""" @@ -193,7 +201,8 @@ class Git(VersionControl): def fetch_new(self, dest, url, rev_options): rev_display = rev_options.to_display() logger.info( - 'Cloning %s%s to %s', url, rev_display, display_path(dest), + 'Cloning %s%s to %s', redact_password_from_url(url), + rev_display, display_path(dest), ) self.run_command(['clone', '-q', url, dest]) @@ -207,7 +216,7 @@ class Git(VersionControl): if not self.is_commit_id_equal(dest, rev_options.rev): cmd_args = ['checkout', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - elif self.get_branch(dest) != branch_name: + elif self.get_current_branch(dest) != branch_name: # Then a specific branch was requested, and that branch # is not yet checked out. track_branch = 'origin/{}'.format(branch_name) @@ -240,14 +249,26 @@ class Git(VersionControl): #: update submodules self.update_submodules(dest) - def get_url(self, location): - """Return URL of the first remote encountered.""" - remotes = self.run_command( + @classmethod + def get_remote_url(cls, location): + """ + Return URL of the first remote encountered. + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + # We need to pass 1 for extra_ok_returncodes since the command + # exits with return code 1 if there are no matching lines. + stdout = cls.run_command( ['config', '--get-regexp', r'remote\..*\.url'], - show_stdout=False, cwd=location, + extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, ) - remotes = remotes.splitlines() - found_remote = remotes[0] + remotes = stdout.splitlines() + try: + found_remote = remotes[0] + except IndexError: + raise RemoteNotFoundError + for remote in remotes: if remote.startswith('remote.origin.url '): found_remote = remote @@ -255,19 +276,21 @@ class Git(VersionControl): url = found_remote.split(' ')[1] return url.strip() - def get_revision(self, location, rev=None): + @classmethod + def get_revision(cls, location, rev=None): if rev is None: rev = 'HEAD' - current_rev = self.run_command( + current_rev = cls.run_command( ['rev-parse', rev], show_stdout=False, cwd=location, ) return current_rev.strip() - def _get_subdirectory(self, location): + @classmethod + def _get_subdirectory(cls, location): """Return the relative path of setup.py to the git repo root.""" # find the repo root - git_dir = self.run_command(['rev-parse', '--git-dir'], - show_stdout=False, cwd=location).strip() + git_dir = cls.run_command(['rev-parse', '--git-dir'], + show_stdout=False, cwd=location).strip() if not os.path.isabs(git_dir): git_dir = os.path.join(location, git_dir) root_dir = os.path.join(git_dir, '..') @@ -290,14 +313,14 @@ class Git(VersionControl): return None return os.path.relpath(location, root_dir) - def get_src_requirement(self, dist, location): - repo = self.get_url(location) + @classmethod + def get_src_requirement(cls, location, project_name): + repo = cls.get_remote_url(location) if not repo.lower().startswith('git:'): repo = 'git+' + repo - current_rev = self.get_revision(location) - egg_project_name = dist.egg_name().split('-', 1)[0] - subdir = self._get_subdirectory(location) - req = make_vcs_requirement_url(repo, current_rev, egg_project_name, + current_rev = cls.get_revision(location) + subdir = cls._get_subdirectory(location) + req = make_vcs_requirement_url(repo, current_rev, project_name, subdir=subdir) return req @@ -332,10 +355,10 @@ class Git(VersionControl): if super(Git, cls).controls_location(location): return True try: - r = cls().run_command(['rev-parse'], - cwd=location, - show_stdout=False, - on_returncode='ignore') + r = cls.run_command(['rev-parse'], + cwd=location, + show_stdout=False, + on_returncode='ignore') return not r except BadCommand: logger.debug("could not determine if %s is under git control " diff --git a/pipenv/patched/notpip/_internal/vcs/mercurial.py b/pipenv/patched/notpip/_internal/vcs/mercurial.py index d76d47f3..37a3f7c2 100644 --- a/pipenv/patched/notpip/_internal/vcs/mercurial.py +++ b/pipenv/patched/notpip/_internal/vcs/mercurial.py @@ -64,34 +64,36 @@ class Mercurial(VersionControl): cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def get_url(self, location): - url = self.run_command( + @classmethod + def get_remote_url(cls, location): + url = cls.run_command( ['showconfig', 'paths.default'], show_stdout=False, cwd=location).strip() - if self._is_local_repository(url): + if cls._is_local_repository(url): url = path_to_url(url) return url.strip() - def get_revision(self, location): - current_revision = self.run_command( + @classmethod + def get_revision(cls, location): + current_revision = cls.run_command( ['parents', '--template={rev}'], show_stdout=False, cwd=location).strip() return current_revision - def get_revision_hash(self, location): - current_rev_hash = self.run_command( + @classmethod + def get_revision_hash(cls, location): + current_rev_hash = cls.run_command( ['parents', '--template={node}'], show_stdout=False, cwd=location).strip() return current_rev_hash - def get_src_requirement(self, dist, location): - repo = self.get_url(location) + @classmethod + def get_src_requirement(cls, location, project_name): + repo = cls.get_remote_url(location) if not repo.lower().startswith('hg:'): repo = 'hg+' + repo - current_rev_hash = self.get_revision_hash(location) - egg_project_name = dist.egg_name().split('-', 1)[0] - return make_vcs_requirement_url(repo, current_rev_hash, - egg_project_name) + current_rev_hash = cls.get_revision_hash(location) + return make_vcs_requirement_url(repo, current_rev_hash, project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/subversion.py b/pipenv/patched/notpip/_internal/vcs/subversion.py index f3c3db4d..e62d3def 100644 --- a/pipenv/patched/notpip/_internal/vcs/subversion.py +++ b/pipenv/patched/notpip/_internal/vcs/subversion.py @@ -4,7 +4,6 @@ import logging import os import re -from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( display_path, make_vcs_requirement_url, rmtree, split_auth_from_netloc, @@ -61,21 +60,8 @@ class Subversion(VersionControl): cmd_args = ['update'] + rev_options.to_args() + [dest] self.run_command(cmd_args) - def get_location(self, dist, dependency_links): - for url in dependency_links: - egg_fragment = Link(url).egg_fragment - if not egg_fragment: - continue - if '-' in egg_fragment: - # FIXME: will this work when a package has - in the name? - key = '-'.join(egg_fragment.split('-')[:-1]).lower() - else: - key = egg_fragment - if key == dist.key: - return url.split('#', 1)[0] - return None - - def get_revision(self, location): + @classmethod + def get_revision(cls, location): """ Return the maximum revision for all files under a given location """ @@ -83,16 +69,16 @@ class Subversion(VersionControl): revision = 0 for base, dirs, files in os.walk(location): - if self.dirname not in dirs: + if cls.dirname not in dirs: dirs[:] = [] continue # no sense walking uncontrolled subdirs - dirs.remove(self.dirname) - entries_fn = os.path.join(base, self.dirname, 'entries') + dirs.remove(cls.dirname) + entries_fn = os.path.join(base, cls.dirname, 'entries') if not os.path.exists(entries_fn): # FIXME: should we warn? continue - dirurl, localrev = self._get_svn_url_rev(base) + dirurl, localrev = cls._get_svn_url_rev(base) if base == location: base = dirurl + '/' # save the root url @@ -131,7 +117,8 @@ class Subversion(VersionControl): return extra_args - def get_url(self, location): + @classmethod + def get_remote_url(cls, location): # In cases where the source is in a subdirectory, not alongside # setup.py we have to look up in the location until we find a real # setup.py @@ -149,12 +136,13 @@ class Subversion(VersionControl): ) return None - return self._get_svn_url_rev(location)[0] + return cls._get_svn_url_rev(location)[0] - def _get_svn_url_rev(self, location): + @classmethod + def _get_svn_url_rev(cls, location): from pipenv.patched.notpip._internal.exceptions import InstallationError - entries_path = os.path.join(location, self.dirname, 'entries') + entries_path = os.path.join(location, cls.dirname, 'entries') if os.path.exists(entries_path): with open(entries_path) as f: data = f.read() @@ -177,7 +165,7 @@ class Subversion(VersionControl): else: try: # subversion >= 1.7 - xml = self.run_command( + xml = cls.run_command( ['info', '--xml', location], show_stdout=False, ) @@ -195,15 +183,14 @@ class Subversion(VersionControl): return url, rev - def get_src_requirement(self, dist, location): - repo = self.get_url(location) + @classmethod + def get_src_requirement(cls, location, project_name): + repo = cls.get_remote_url(location) if repo is None: return None repo = 'svn+' + repo - rev = self.get_revision(location) - # FIXME: why not project name? - egg_project_name = dist.egg_name().split('-', 1)[0] - return make_vcs_requirement_url(repo, rev, egg_project_name) + rev = cls.get_revision(location) + return make_vcs_requirement_url(repo, rev, project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/wheel.py b/pipenv/patched/notpip/_internal/wheel.py index 6df5a3a3..06c880e8 100644 --- a/pipenv/patched/notpip/_internal/wheel.py +++ b/pipenv/patched/notpip/_internal/wheel.py @@ -30,6 +30,7 @@ from pipenv.patched.notpip._internal.exceptions import ( from pipenv.patched.notpip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, distutils_scheme, ) +from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( call_subprocess, captured_stdout, ensure_dir, read_chunks, @@ -40,9 +41,22 @@ from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import open_spinner if MYPY_CHECK_RUNNING: - from typing import Dict, List, Optional # noqa: F401 + from typing import ( # noqa: F401 + Dict, List, Optional, Sequence, Mapping, Tuple, IO, Text, Any, + Union, Iterable + ) + from pipenv.patched.notpip._vendor.packaging.requirements import Requirement # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + from pipenv.patched.notpip._internal.index import FormatControl, PackageFinder # noqa: F401 + from pipenv.patched.notpip._internal.operations.prepare import ( # noqa: F401 + RequirementPreparer + ) + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + from pipenv.patched.notpip._internal.pep425tags import Pep425Tag # noqa: F401 + + InstalledCSVRow = Tuple[str, ...] -wheel_ext = '.whl' VERSION_COMPATIBLE = (1, 0) @@ -50,7 +64,12 @@ VERSION_COMPATIBLE = (1, 0) logger = logging.getLogger(__name__) +def normpath(src, p): + return os.path.relpath(src, p).replace(os.path.sep, '/') + + def rehash(path, blocksize=1 << 20): + # type: (str, int) -> Tuple[str, str] """Return (hash, length) for path using hashlib.sha256()""" h = hashlib.sha256() length = 0 @@ -61,20 +80,32 @@ def rehash(path, blocksize=1 << 20): digest = 'sha256=' + urlsafe_b64encode( h.digest() ).decode('latin1').rstrip('=') - return (digest, length) + # unicode/str python2 issues + return (digest, str(length)) # type: ignore def open_for_csv(name, mode): + # type: (str, Text) -> IO if sys.version_info[0] < 3: - nl = {} + nl = {} # type: Dict[str, Any] bin = 'b' else: - nl = {'newline': ''} + nl = {'newline': ''} # type: Dict[str, Any] bin = '' return open(name, mode + bin, **nl) +def replace_python_tag(wheelname, new_tag): + # type: (str, str) -> str + """Replace the Python tag in a wheel file name with a new value. + """ + parts = wheelname.split('-') + parts[-3] = new_tag + return '-'.join(parts) + + def fix_script(path): + # type: (str) -> Optional[bool] """Replace #!python with #!/path/to/python Return True if file was changed.""" # XXX RECORD hashes will need to be updated @@ -90,6 +121,7 @@ def fix_script(path): script.write(firstline) script.write(rest) return True + return None dist_info_re = re.compile(r"""^(?P(?P.+?)(-(?P.+?))?) @@ -97,6 +129,7 @@ dist_info_re = re.compile(r"""^(?P(?P.+?)(-(?P.+?))?) def root_is_purelib(name, wheeldir): + # type: (str, str) -> bool """ Return True if the extracted wheel in wheeldir should go into purelib. """ @@ -113,6 +146,7 @@ def root_is_purelib(name, wheeldir): def get_entrypoints(filename): + # type: (str) -> Tuple[Dict[str, str], Dict[str, str]] if not os.path.exists(filename): return {}, {} @@ -144,7 +178,7 @@ def get_entrypoints(filename): def message_about_scripts_not_on_PATH(scripts): - # type: (List[str]) -> Optional[str] + # type: (Sequence[str]) -> Optional[str] """Determine if any scripts are not on PATH and format a warning. Returns a warning message if one or more scripts are not on PATH, @@ -205,10 +239,81 @@ def message_about_scripts_not_on_PATH(scripts): return "\n".join(msg_lines) -def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, - pycompile=True, scheme=None, isolated=False, prefix=None, - warn_script_location=True): +def sorted_outrows(outrows): + # type: (Iterable[InstalledCSVRow]) -> List[InstalledCSVRow] + """ + Return the given rows of a RECORD file in sorted order. + + Each row is a 3-tuple (path, hash, size) and corresponds to a record of + a RECORD file (see PEP 376 and PEP 427 for details). For the rows + passed to this function, the size can be an integer as an int or string, + or the empty string. + """ + # Normally, there should only be one row per path, in which case the + # second and third elements don't come into play when sorting. + # However, in cases in the wild where a path might happen to occur twice, + # we don't want the sort operation to trigger an error (but still want + # determinism). Since the third element can be an int or string, we + # coerce each element to a string to avoid a TypeError in this case. + # For additional background, see-- + # https://github.com/pypa/pip/issues/5868 + return sorted(outrows, key=lambda row: tuple(str(x) for x in row)) + + +def get_csv_rows_for_installed( + old_csv_rows, # type: Iterable[List[str]] + installed, # type: Dict[str, str] + changed, # type: set + generated, # type: List[str] + lib_dir, # type: str +): + # type: (...) -> List[InstalledCSVRow] + """ + :param installed: A map from archive RECORD path to installation RECORD + path. + """ + installed_rows = [] # type: List[InstalledCSVRow] + for row in old_csv_rows: + if len(row) > 3: + logger.warning( + 'RECORD line has more than three elements: {}'.format(row) + ) + # Make a copy because we are mutating the row. + row = list(row) + old_path = row[0] + new_path = installed.pop(old_path, old_path) + row[0] = new_path + if new_path in changed: + digest, length = rehash(new_path) + row[1] = digest + row[2] = length + installed_rows.append(tuple(row)) + for f in generated: + digest, length = rehash(f) + installed_rows.append((normpath(f, lib_dir), digest, str(length))) + for f in installed: + installed_rows.append((installed[f], '', '')) + return installed_rows + + +def move_wheel_files( + name, # type: str + req, # type: Requirement + wheeldir, # type: str + user=False, # type: bool + home=None, # type: Optional[str] + root=None, # type: Optional[str] + pycompile=True, # type: bool + scheme=None, # type: Optional[Mapping[str, str]] + isolated=False, # type: bool + prefix=None, # type: Optional[str] + warn_script_location=True # type: bool +): + # type: (...) -> None """Install a wheel""" + # TODO: Investigate and break this up. + # TODO: Look into moving this into a dedicated class for representing an + # installation. if not scheme: scheme = distutils_scheme( @@ -221,7 +326,7 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, else: lib_dir = scheme['platlib'] - info_dir = [] + info_dir = [] # type: List[str] data_dirs = [] source = wheeldir.rstrip(os.path.sep) + os.path.sep @@ -229,9 +334,9 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, # installed = files copied from the wheel to the destination # changed = files changed while installing (scripts #! line typically) # generated = files newly generated during the install (script wrappers) - installed = {} + installed = {} # type: Dict[str, str] changed = set() - generated = [] + generated = [] # type: List[str] # Compile all of the pyc files that we're going to be installing if pycompile: @@ -241,9 +346,6 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, compileall.compile_dir(source, force=True, quiet=True) logger.debug(stdout.getvalue()) - def normpath(src, p): - return os.path.relpath(src, p).replace(os.path.sep, '/') - def record_installed(srcfile, destfile, modified=False): """Map archive RECORD paths to installation RECORD paths.""" oldpath = normpath(srcfile, wheeldir) @@ -389,8 +491,9 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, "import_name": entry.suffix.split(".")[0], "func": entry.suffix, } - - maker._get_script_text = _get_script_text + # ignore type, because mypy disallows assigning to a method, + # see https://github.com/python/mypy/issues/2427 + maker._get_script_text = _get_script_text # type: ignore maker.script_template = r"""# -*- coding: utf-8 -*- import re import sys @@ -500,28 +603,23 @@ if __name__ == '__main__': with open_for_csv(record, 'r') as record_in: with open_for_csv(temp_record, 'w+') as record_out: reader = csv.reader(record_in) + outrows = get_csv_rows_for_installed( + reader, installed=installed, changed=changed, + generated=generated, lib_dir=lib_dir, + ) writer = csv.writer(record_out) - outrows = [] - for row in reader: - row[0] = installed.pop(row[0], row[0]) - if row[0] in changed: - row[1], row[2] = rehash(row[0]) - outrows.append(tuple(row)) - for f in generated: - digest, length = rehash(f) - outrows.append((normpath(f, lib_dir), digest, length)) - for f in installed: - outrows.append((installed[f], '', '')) - for row in sorted(outrows): + # Sort to simplify testing. + for row in sorted_outrows(outrows): writer.writerow(row) shutil.move(temp_record, record) def wheel_version(source_dir): + # type: (Optional[str]) -> Optional[Tuple[int, ...]] """ Return the Wheel-Version of an extracted wheel, if possible. - Otherwise, return False if we couldn't parse / extract it. + Otherwise, return None if we couldn't parse / extract it. """ try: dist = [d for d in pkg_resources.find_on_path(None, source_dir)][0] @@ -533,10 +631,11 @@ def wheel_version(source_dir): version = tuple(map(int, version.split('.'))) return version except Exception: - return False + return None def check_compatibility(version, name): + # type: (Optional[Tuple[int, ...]], str) -> None """ Raises errors or warns if called with an incompatible Wheel-Version. @@ -568,7 +667,8 @@ def check_compatibility(version, name): class Wheel(object): """A wheel file""" - # TODO: maybe move the install code into this class + # TODO: Maybe move the class into the models sub-package + # TODO: Maybe move the install code into this class wheel_file_re = re.compile( r"""^(?P(?P.+?)-(?P.*?)) @@ -578,6 +678,7 @@ class Wheel(object): ) def __init__(self, filename): + # type: (str) -> None """ :raises InvalidWheelFilename: when the filename is invalid for a wheel """ @@ -603,6 +704,7 @@ class Wheel(object): } def support_index_min(self, tags=None): + # type: (Optional[List[Pep425Tag]]) -> Optional[int] """ Return the lowest index that one of the wheel's file_tag combinations achieves in the supported_tags list e.g. if there are 8 supported tags, @@ -615,17 +717,146 @@ class Wheel(object): return min(indexes) if indexes else None def supported(self, tags=None): + # type: (Optional[List[Pep425Tag]]) -> bool """Is this wheel supported on this system?""" if tags is None: # for mock tags = pep425tags.get_supported() return bool(set(tags).intersection(self.file_tags)) +def _contains_egg_info( + s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): + """Determine whether the string looks like an egg_info. + + :param s: The string to parse. E.g. foo-2.1 + """ + return bool(_egg_info_re.search(s)) + + +def should_use_ephemeral_cache( + req, # type: InstallRequirement + format_control, # type: FormatControl + autobuilding, # type: bool + cache_available # type: bool +): + # type: (...) -> Optional[bool] + """ + Return whether to build an InstallRequirement object using the + ephemeral cache. + + :param cache_available: whether a cache directory is available for the + autobuilding=True case. + + :return: True or False to build the requirement with ephem_cache=True + or False, respectively; or None not to build the requirement. + """ + if req.constraint: + return None + if req.is_wheel: + if not autobuilding: + logger.info( + 'Skipping %s, due to already being wheel.', req.name, + ) + return None + if not autobuilding: + return False + + if req.editable or not req.source_dir: + return None + + if req.link and not req.link.is_artifact: + # VCS checkout. Build wheel just for this run. + return True + + if "binary" not in format_control.get_allowed_formats( + canonicalize_name(req.name)): + logger.info( + "Skipping bdist_wheel for %s, due to binaries " + "being disabled for it.", req.name, + ) + return None + + link = req.link + base, ext = link.splitext() + if cache_available and _contains_egg_info(base): + return False + + # Otherwise, build the wheel just for this run using the ephemeral + # cache since we are either in the case of e.g. a local directory, or + # no cache directory is available to use. + return True + + +def format_command( + command_args, # type: List[str] + command_output, # type: str +): + # type: (...) -> str + """ + Format command information for logging. + """ + text = 'Command arguments: {}\n'.format(command_args) + + if not command_output: + text += 'Command output: None' + elif logger.getEffectiveLevel() > logging.DEBUG: + text += 'Command output: [use --verbose to show]' + else: + if not command_output.endswith('\n'): + command_output += '\n' + text += ( + 'Command output:\n{}' + '-----------------------------------------' + ).format(command_output) + + return text + + +def get_legacy_build_wheel_path( + names, # type: List[str] + temp_dir, # type: str + req, # type: InstallRequirement + command_args, # type: List[str] + command_output, # type: str +): + # type: (...) -> Optional[str] + """ + Return the path to the wheel in the temporary build directory. + """ + # Sort for determinism. + names = sorted(names) + if not names: + msg = ( + 'Legacy build of wheel for {!r} created no files.\n' + ).format(req.name) + msg += format_command(command_args, command_output) + logger.warning(msg) + return None + + if len(names) > 1: + msg = ( + 'Legacy build of wheel for {!r} created more than one file.\n' + 'Filenames (choosing first): {}\n' + ).format(req.name, names) + msg += format_command(command_args, command_output) + logger.warning(msg) + + return os.path.join(temp_dir, names[0]) + + class WheelBuilder(object): """Build wheels from a RequirementSet.""" - def __init__(self, finder, preparer, wheel_cache, - build_options=None, global_options=None, no_clean=False): + def __init__( + self, + finder, # type: PackageFinder + preparer, # type: RequirementPreparer + wheel_cache, # type: WheelCache + build_options=None, # type: Optional[List[str]] + global_options=None, # type: Optional[List[str]] + no_clean=False # type: bool + ): + # type: (...) -> None self.finder = finder self.preparer = preparer self.wheel_cache = wheel_cache @@ -648,15 +879,18 @@ class WheelBuilder(object): def _build_one_inside_env(self, req, output_dir, python_tag=None): with TempDirectory(kind="wheel") as temp_dir: - if self.__build_one(req, temp_dir.path, python_tag=python_tag): + if req.use_pep517: + builder = self._build_one_pep517 + else: + builder = self._build_one_legacy + wheel_path = builder(req, temp_dir.path, python_tag=python_tag) + if wheel_path is not None: + wheel_name = os.path.basename(wheel_path) + dest_path = os.path.join(output_dir, wheel_name) try: - wheel_name = os.listdir(temp_dir.path)[0] - wheel_path = os.path.join(output_dir, wheel_name) - shutil.move( - os.path.join(temp_dir.path, wheel_name), wheel_path - ) + shutil.move(wheel_path, dest_path) logger.info('Stored in directory: %s', output_dir) - return wheel_path + return dest_path except Exception: pass # Ignore return, we can't do anything else useful. @@ -674,10 +908,43 @@ class WheelBuilder(object): SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) - def __build_one(self, req, tempd, python_tag=None): + def _build_one_pep517(self, req, tempd, python_tag=None): + """Build one InstallRequirement using the PEP 517 build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ + assert req.metadata_directory is not None + try: + req.spin_message = 'Building wheel for %s (PEP 517)' % (req.name,) + logger.debug('Destination directory: %s', tempd) + wheel_name = req.pep517_backend.build_wheel( + tempd, + metadata_directory=req.metadata_directory + ) + if python_tag: + # General PEP 517 backends don't necessarily support + # a "--python-tag" option, so we rename the wheel + # file directly. + new_name = replace_python_tag(wheel_name, python_tag) + os.rename( + os.path.join(tempd, wheel_name), + os.path.join(tempd, new_name) + ) + # Reassign to simplify the return at the end of function + wheel_name = new_name + except Exception: + logger.error('Failed building wheel for %s', req.name) + return None + return os.path.join(tempd, wheel_name) + + def _build_one_legacy(self, req, tempd, python_tag=None): + """Build one InstallRequirement using the "legacy" build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ base_args = self._base_setup_args(req) - spin_message = 'Running setup.py bdist_wheel for %s' % (req.name,) + spin_message = 'Building wheel for %s (setup.py)' % (req.name,) with open_spinner(spin_message) as spinner: logger.debug('Destination directory: %s', tempd) wheel_args = base_args + ['bdist_wheel', '-d', tempd] \ @@ -687,13 +954,21 @@ class WheelBuilder(object): wheel_args += ["--python-tag", python_tag] try: - call_subprocess(wheel_args, cwd=req.setup_py_dir, - show_stdout=False, spinner=spinner) - return True + output = call_subprocess(wheel_args, cwd=req.setup_py_dir, + show_stdout=False, spinner=spinner) except Exception: spinner.finish("error") logger.error('Failed building wheel for %s', req.name) - return False + return None + names = os.listdir(tempd) + wheel_path = get_legacy_build_wheel_path( + names=names, + temp_dir=tempd, + req=req, + command_args=wheel_args, + command_output=output, + ) + return wheel_path def _clean_one(self, req): base_args = self._base_setup_args(req) @@ -707,57 +982,46 @@ class WheelBuilder(object): logger.error('Failed cleaning build dir for %s', req.name) return False - def build(self, requirements, session, autobuilding=False): + def build( + self, + requirements, # type: Iterable[InstallRequirement] + session, # type: PipSession + autobuilding=False # type: bool + ): + # type: (...) -> List[InstallRequirement] """Build wheels. :param unpack: If True, replace the sdist we built from with the newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ - from pipenv.patched.notpip._internal import index - from pipenv.patched.notpip._internal.models.link import Link - - building_is_possible = self._wheel_dir or ( - autobuilding and self.wheel_cache.cache_dir - ) - assert building_is_possible - buildset = [] format_control = self.finder.format_control + # Whether a cache directory is available for autobuilding=True. + cache_available = bool(self._wheel_dir or self.wheel_cache.cache_dir) + for req in requirements: - if req.constraint: + ephem_cache = should_use_ephemeral_cache( + req, format_control=format_control, autobuilding=autobuilding, + cache_available=cache_available, + ) + if ephem_cache is None: continue - if req.is_wheel: - if not autobuilding: - logger.info( - 'Skipping %s, due to already being wheel.', req.name, - ) - elif autobuilding and req.editable: - pass - elif autobuilding and not req.source_dir: - pass - elif autobuilding and req.link and not req.link.is_artifact: - # VCS checkout. Build wheel just for this run. - buildset.append((req, True)) - else: - ephem_cache = False - if autobuilding: - link = req.link - base, ext = link.splitext() - if index.egg_info_matches(base, None, link) is None: - # E.g. local directory. Build wheel just for this run. - ephem_cache = True - if "binary" not in format_control.get_allowed_formats( - canonicalize_name(req.name)): - logger.info( - "Skipping bdist_wheel for %s, due to binaries " - "being disabled for it.", req.name, - ) - continue - buildset.append((req, ephem_cache)) + + buildset.append((req, ephem_cache)) if not buildset: - return True + return [] + + # Is any wheel build not using the ephemeral cache? + if any(not ephem_cache for _, ephem_cache in buildset): + have_directory_for_build = self._wheel_dir or ( + autobuilding and self.wheel_cache.cache_dir + ) + assert have_directory_for_build + + # TODO by @pradyunsg + # Should break up this method into 2 separate methods. # Build the wheels. logger.info( @@ -829,5 +1093,5 @@ class WheelBuilder(object): 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) - # Return True if all builds were successful - return len(build_failure) == 0 + # Return a list of requirements that failed to build + return build_failure diff --git a/pipenv/patched/notpip/_vendor/__init__.py b/pipenv/patched/notpip/_vendor/__init__.py index b6294b21..1256c039 100644 --- a/pipenv/patched/notpip/_vendor/__init__.py +++ b/pipenv/patched/notpip/_vendor/__init__.py @@ -74,6 +74,7 @@ if DEBUNDLED: vendored("packaging") vendored("packaging.version") vendored("packaging.specifiers") + vendored("pep517") vendored("pkg_resources") vendored("progress") vendored("pytoml") diff --git a/pipenv/patched/notpip/_vendor/certifi/__init__.py b/pipenv/patched/notpip/_vendor/certifi/__init__.py index aa329fbb..ef71f3af 100644 --- a/pipenv/patched/notpip/_vendor/certifi/__init__.py +++ b/pipenv/patched/notpip/_vendor/certifi/__init__.py @@ -1,3 +1,3 @@ -from .core import where, old_where +from .core import where -__version__ = "2018.08.24" +__version__ = "2018.11.29" diff --git a/pipenv/patched/notpip/_vendor/certifi/cacert.pem b/pipenv/patched/notpip/_vendor/certifi/cacert.pem index 85de024e..db68797e 100644 --- a/pipenv/patched/notpip/_vendor/certifi/cacert.pem +++ b/pipenv/patched/notpip/_vendor/certifi/cacert.pem @@ -326,36 +326,6 @@ OCiNUW7dFGdTbHFcJoRNdVq2fmBWqU2t+5sel/MN2dKXVHfaPRK34B7vCAas+YWH QMAJKOSLakhT2+zNVVXxxvjpoixMptEmX36vWkzaH6byHCx+rgIW0lbQL1dTR+iS -----END CERTIFICATE----- -# Issuer: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Subject: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Label: "Visa eCommerce Root" -# Serial: 25952180776285836048024890241505565794 -# MD5 Fingerprint: fc:11:b8:d8:08:93:30:00:6d:23:f9:7e:eb:52:1e:02 -# SHA1 Fingerprint: 70:17:9b:86:8c:00:a4:fa:60:91:52:22:3f:9f:3e:32:bd:e0:05:62 -# SHA256 Fingerprint: 69:fa:c9:bd:55:fb:0a:c7:8d:53:bb:ee:5c:f1:d5:97:98:9f:d0:aa:ab:20:a2:51:51:bd:f1:73:3e:e7:d1:22 ------BEGIN CERTIFICATE----- -MIIDojCCAoqgAwIBAgIQE4Y1TR0/BvLB+WUF1ZAcYjANBgkqhkiG9w0BAQUFADBr -MQswCQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRl -cm5hdGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNv -bW1lcmNlIFJvb3QwHhcNMDIwNjI2MDIxODM2WhcNMjIwNjI0MDAxNjEyWjBrMQsw -CQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRlcm5h -dGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNvbW1l -cmNlIFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvV95WHm6h -2mCxlCfLF9sHP4CFT8icttD0b0/Pmdjh28JIXDqsOTPHH2qLJj0rNfVIsZHBAk4E -lpF7sDPwsRROEW+1QK8bRaVK7362rPKgH1g/EkZgPI2h4H3PVz4zHvtH8aoVlwdV -ZqW1LS7YgFmypw23RuwhY/81q6UCzyr0TP579ZRdhE2o8mCP2w4lPJ9zcc+U30rq -299yOIzzlr3xF7zSujtFWsan9sYXiwGd/BmoKoMWuDpI/k4+oKsGGelT84ATB+0t -vz8KPFUgOSwsAGl0lUq8ILKpeeUYiZGo3BxN77t+Nwtd/jmliFKMAGzsGHxBvfaL -dXe6YJ2E5/4tAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD -AgEGMB0GA1UdDgQWBBQVOIMPPyw/cDMezUb+B4wg4NfDtzANBgkqhkiG9w0BAQUF -AAOCAQEAX/FBfXxcCLkr4NWSR/pnXKUTwwMhmytMiUbPWU3J/qVAtmPN3XEolWcR -zCSs00Rsca4BIGsDoo8Ytyk6feUWYFN4PMCvFYP3j1IzJL1kk5fui/fbGKhtcbP3 -LBfQdCVp9/5rPJS+TUtBjE7ic9DjkCJzQ83z7+pzzkWKsKZJ/0x9nXGIxHYdkFsd -7v3M9+79YKWxehZx0RbQfBI8bGmX265fOZpwLwU8GUYEmSA20GBuYQa7FkKMcPcw -++DbZqMAAb3mLNqRX6BGi01qnD093QVG/na/oAo85ADmJ7f/hC3euiInlhBx6yLt -398znM/jra6O1I7mT1GvFpLgXPYHDw== ------END CERTIFICATE----- - # Issuer: CN=AAA Certificate Services O=Comodo CA Limited # Subject: CN=AAA Certificate Services O=Comodo CA Limited # Label: "Comodo AAA Services root" @@ -4298,3 +4268,245 @@ rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV 57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 -----END CERTIFICATE----- + +# Issuer: CN=GTS Root R1 O=Google Trust Services LLC +# Subject: CN=GTS Root R1 O=Google Trust Services LLC +# Label: "GTS Root R1" +# Serial: 146587175971765017618439757810265552097 +# MD5 Fingerprint: 82:1a:ef:d4:d2:4a:f2:9f:e2:3d:97:06:14:70:72:85 +# SHA1 Fingerprint: e1:c9:50:e6:ef:22:f8:4c:56:45:72:8b:92:20:60:d7:d5:a7:a3:e8 +# SHA256 Fingerprint: 2a:57:54:71:e3:13:40:bc:21:58:1c:bd:2c:f1:3e:15:84:63:20:3e:ce:94:bc:f9:d3:cc:19:6b:f0:9a:54:72 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxUtHDA3sM9CJuRz04TANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaM +f/vo27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vX +mX7wCl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7 +zUjwTcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0P +fyblqAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtc +vfaHszVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4 +Zor8Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUsp +zBmkMiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOO +Rc92wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYW +k70paDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+ +DVrNVjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgF +lQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBADiW +Cu49tJYeX++dnAsznyvgyv3SjgofQXSlfKqE1OXyHuY3UjKcC9FhHb8owbZEKTV1 +d5iyfNm9dKyKaOOpMQkpAWBz40d8U6iQSifvS9efk+eCNs6aaAyC58/UEBZvXw6Z +XPYfcX3v73svfuo21pdwCxXu11xWajOl40k4DLh9+42FpLFZXvRq4d2h9mREruZR +gyFmxhE+885H7pwoHyXa/6xmld01D1zvICxi/ZG6qcz8WpyTgYMpl0p8WnK0OdC3 +d8t5/Wk6kjftbjhlRn7pYL15iJdfOBL07q9bgsiG1eGZbYwE8na6SfZu6W0eX6Dv +J4J2QPim01hcDyxC2kLGe4g0x8HYRZvBPsVhHdljUEn2NIVq4BjFbkerQUIpm/Zg +DdIx02OYI5NaAIFItO/Nis3Jz5nu2Z6qNuFoS3FJFDYoOj0dzpqPJeaAcWErtXvM ++SUWgeExX6GjfhaknBZqlxi9dnKlC54dNuYvoS++cJEPqOba+MSSQGwlfnuzCdyy +F62ARPBopY+Udf90WuioAnwMCeKpSwughQtiue+hMZL77/ZRBIls6Kl0obsXs7X9 +SQ98POyDGCBDTtWTurQ0sR8WNh8M5mQ5Fkzc4P4dyKliPUDqysU0ArSuiYgzNdws +E3PYJ/HQcu51OyLemGhmW/HGY0dVHLqlCFF1pkgl +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R2 O=Google Trust Services LLC +# Subject: CN=GTS Root R2 O=Google Trust Services LLC +# Label: "GTS Root R2" +# Serial: 146587176055767053814479386953112547951 +# MD5 Fingerprint: 44:ed:9a:0e:a4:09:3b:00:f2:ae:4c:a3:c6:61:b0:8b +# SHA1 Fingerprint: d2:73:96:2a:2a:5e:39:9f:73:3f:e1:c7:1e:64:3f:03:38:34:fc:4d +# SHA256 Fingerprint: c4:5d:7b:b0:8e:6d:67:e6:2e:42:35:11:0b:56:4e:5f:78:fd:92:ef:05:8c:84:0a:ea:4e:64:55:d7:58:5c:60 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxlqz5yDFMJo/aFLybzANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3Lv +CvptnfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3Kg +GjSY6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9Bu +XvAuMC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOd +re7kRXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXu +PuWgf9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1 +mKPV+3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K +8YzodDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqj +x5RWIr9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsR +nTKaG73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0 +kzCqgc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9Ok +twIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBALZp +8KZ3/p7uC4Gt4cCpx/k1HUCCq+YEtN/L9x0Pg/B+E02NjO7jMyLDOfxA325BS0JT +vhaI8dI4XsRomRyYUpOM52jtG2pzegVATX9lO9ZY8c6DR2Dj/5epnGB3GFW1fgiT +z9D2PGcDFWEJ+YF59exTpJ/JjwGLc8R3dtyDovUMSRqodt6Sm2T4syzFJ9MHwAiA +pJiS4wGWAqoC7o87xdFtCjMwc3i5T1QWvwsHoaRc5svJXISPD+AVdyx+Jn7axEvb +pxZ3B7DNdehyQtaVhJ2Gg/LkkM0JR9SLA3DaWsYDQvTtN6LwG1BUSw7YhN4ZKJmB +R64JGz9I0cNv4rBgF/XuIwKl2gBbbZCr7qLpGzvpx0QnRY5rn/WkhLx3+WuXrD5R +RaIRpsyF7gpo8j5QOHokYh4XIDdtak23CZvJ/KRY9bb7nE4Yu5UC56GtmwfuNmsk +0jmGwZODUNKBRqhfYlcsu2xkiAhu7xNUX90txGdj08+JN7+dIPT7eoOboB6BAFDC +5AwiWVIQ7UNWhwD4FFKnHYuTjKJNRn8nxnGbJN7k2oaLDX5rIMHAnuFl2GqjpuiF +izoHCBy69Y9Vmhh1fuXsgWbRIXOhNUQLgD1bnF5vKheW0YMjiGZt5obicDIvUiLn +yOd/xCxgXS/Dr55FBcOEArf9LAhST4Ldo/DUhgkC +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R3 O=Google Trust Services LLC +# Subject: CN=GTS Root R3 O=Google Trust Services LLC +# Label: "GTS Root R3" +# Serial: 146587176140553309517047991083707763997 +# MD5 Fingerprint: 1a:79:5b:6b:04:52:9c:5d:c7:74:33:1b:25:9a:f9:25 +# SHA1 Fingerprint: 30:d4:24:6f:07:ff:db:91:89:8a:0b:e9:49:66:11:eb:8c:5e:46:e5 +# SHA256 Fingerprint: 15:d5:b8:77:46:19:ea:7d:54:ce:1c:a6:d0:b0:c4:03:e0:37:a9:17:f1:31:e8:a0:4e:1e:6b:7a:71:ba:bc:e5 +-----BEGIN CERTIFICATE----- +MIICDDCCAZGgAwIBAgIQbkepx2ypcyRAiQ8DVd2NHTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout +736GjOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2A +DDL24CejQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEAgFuk +fCPAlaUs3L6JbyO5o91lAFJekazInXJ0glMLfalAvWhgxeG4VDvBNhcl2MG9AjEA +njWSdIUlUfUk7GRSJFClH9voy8l27OyCbvWFGFPouOOaKaqW04MjyaR7YbPMAuhd +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R4 O=Google Trust Services LLC +# Subject: CN=GTS Root R4 O=Google Trust Services LLC +# Label: "GTS Root R4" +# Serial: 146587176229350439916519468929765261721 +# MD5 Fingerprint: 5d:b6:6a:c4:60:17:24:6a:1a:99:a8:4b:ee:5e:b4:26 +# SHA1 Fingerprint: 2a:1d:60:27:d9:4a:b1:0a:1c:4d:91:5c:cd:33:a0:cb:3e:2d:54:cb +# SHA256 Fingerprint: 71:cc:a5:39:1f:9e:79:4b:04:80:25:30:b3:63:e1:21:da:8a:30:43:bb:26:66:2f:ea:4d:ca:7f:c9:51:a4:bd +-----BEGIN CERTIFICATE----- +MIICCjCCAZGgAwIBAgIQbkepyIuUtui7OyrYorLBmTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzu +hXyiQHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/l +xKvRHYqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNnADBkAjBqUFJ0 +CMRw3J5QdCHojXohw0+WbhXRIjVhLfoIN+4Zba3bssx9BzT1YBkstTTZbyACMANx +sbqjYAuG7ZoIapVon+Kz4ZNkfF6Tpt95LY2F45TPI11xzPKwTdb+mciUqXWi4w== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Global G2 Root O=UniTrust +# Subject: CN=UCA Global G2 Root O=UniTrust +# Label: "UCA Global G2 Root" +# Serial: 124779693093741543919145257850076631279 +# MD5 Fingerprint: 80:fe:f0:c4:4a:f0:5c:62:32:9f:1c:ba:78:a9:50:f8 +# SHA1 Fingerprint: 28:f9:78:16:19:7a:ff:18:25:18:aa:44:fe:c1:a0:ce:5c:b6:4c:8a +# SHA256 Fingerprint: 9b:ea:11:c9:76:fe:01:47:64:c1:be:56:a6:f9:14:b5:a5:60:31:7a:bd:99:88:39:33:82:e5:16:1a:a0:49:3c +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIQXd+x2lqj7V2+WmUgZQOQ7zANBgkqhkiG9w0BAQsFADA9 +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxGzAZBgNVBAMMElVDQSBH +bG9iYWwgRzIgUm9vdDAeFw0xNjAzMTEwMDAwMDBaFw00MDEyMzEwMDAwMDBaMD0x +CzAJBgNVBAYTAkNOMREwDwYDVQQKDAhVbmlUcnVzdDEbMBkGA1UEAwwSVUNBIEds +b2JhbCBHMiBSb290MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeYr +b3zvJgUno4Ek2m/LAfmZmqkywiKHYUGRO8vDaBsGxUypK8FnFyIdK+35KYmToni9 +kmugow2ifsqTs6bRjDXVdfkX9s9FxeV67HeToI8jrg4aA3++1NDtLnurRiNb/yzm +VHqUwCoV8MmNsHo7JOHXaOIxPAYzRrZUEaalLyJUKlgNAQLx+hVRZ2zA+te2G3/R +VogvGjqNO7uCEeBHANBSh6v7hn4PJGtAnTRnvI3HLYZveT6OqTwXS3+wmeOwcWDc +C/Vkw85DvG1xudLeJ1uK6NjGruFZfc8oLTW4lVYa8bJYS7cSN8h8s+1LgOGN+jIj +tm+3SJUIsUROhYw6AlQgL9+/V087OpAh18EmNVQg7Mc/R+zvWr9LesGtOxdQXGLY +D0tK3Cv6brxzks3sx1DoQZbXqX5t2Okdj4q1uViSukqSKwxW/YDrCPBeKW4bHAyv +j5OJrdu9o54hyokZ7N+1wxrrFv54NkzWbtA+FxyQF2smuvt6L78RHBgOLXMDj6Dl +NaBa4kx1HXHhOThTeEDMg5PXCp6dW4+K5OXgSORIskfNTip1KnvyIvbJvgmRlld6 +iIis7nCs+dwp4wwcOxJORNanTrAmyPPZGpeRaOrvjUYG0lZFWJo8DA+DuAUlwznP +O6Q0ibd5Ei9Hxeepl2n8pndntd978XplFeRhVmUCAwEAAaNCMEAwDgYDVR0PAQH/ +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIHEjMz15DD/pQwIX4wV +ZyF0Ad/fMA0GCSqGSIb3DQEBCwUAA4ICAQATZSL1jiutROTL/7lo5sOASD0Ee/oj +L3rtNtqyzm325p7lX1iPyzcyochltq44PTUbPrw7tgTQvPlJ9Zv3hcU2tsu8+Mg5 +1eRfB70VVJd0ysrtT7q6ZHafgbiERUlMjW+i67HM0cOU2kTC5uLqGOiiHycFutfl +1qnN3e92mI0ADs0b+gO3joBYDic/UvuUospeZcnWhNq5NXHzJsBPd+aBJ9J3O5oU +b3n09tDh05S60FdRvScFDcH9yBIw7m+NESsIndTUv4BFFJqIRNow6rSn4+7vW4LV +PtateJLbXDzz2K36uGt/xDYotgIVilQsnLAXc47QN6MUPJiVAAwpBVueSUmxX8fj +y88nZY41F7dXyDDZQVu5FLbowg+UMaeUmMxq67XhJ/UQqAHojhJi6IjMtX9Gl8Cb +EGY4GjZGXyJoPd/JxhMnq1MGrKI8hgZlb7F+sSlEmqO6SWkoaY/X5V+tBIZkbxqg +DMUIYs6Ao9Dz7GjevjPHF1t/gMRMTLGmhIrDO7gJzRSBuhjjVFc2/tsvfEehOjPI ++Vg7RE+xygKJBJYoaMVLuCaJu9YzL1DV/pqJuhgyklTGW+Cd+V7lDSKb9triyCGy +YiGqhkCyLmTTX8jjfhFnRR8F/uOi77Oos/N9j/gMHyIfLXC0uAE0djAA5SN4p1bX +UB+K+wb1whnw0A== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Extended Validation Root O=UniTrust +# Subject: CN=UCA Extended Validation Root O=UniTrust +# Label: "UCA Extended Validation Root" +# Serial: 106100277556486529736699587978573607008 +# MD5 Fingerprint: a1:f3:5f:43:c6:34:9b:da:bf:8c:7e:05:53:ad:96:e2 +# SHA1 Fingerprint: a3:a1:b0:6f:24:61:23:4a:e3:36:a5:c2:37:fc:a6:ff:dd:f0:d7:3a +# SHA256 Fingerprint: d4:3a:f9:b3:54:73:75:5c:96:84:fc:06:d7:d8:cb:70:ee:5c:28:e7:73:fb:29:4e:b4:1e:e7:17:22:92:4d:24 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQT9Irj/VkyDOeTzRYZiNwYDANBgkqhkiG9w0BAQsFADBH +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNVBAMMHFVDQSBF +eHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwHhcNMTUwMzEzMDAwMDAwWhcNMzgxMjMx +MDAwMDAwWjBHMQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNV +BAMMHFVDQSBFeHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCpCQcoEwKwmeBkqh5DFnpzsZGgdT6o+uM4AHrsiWog +D4vFsJszA1qGxliG1cGFu0/GnEBNyr7uaZa4rYEwmnySBesFK5pI0Lh2PpbIILvS +sPGP2KxFRv+qZ2C0d35qHzwaUnoEPQc8hQ2E0B92CvdqFN9y4zR8V05WAT558aop +O2z6+I9tTcg1367r3CTueUWnhbYFiN6IXSV8l2RnCdm/WhUFhvMJHuxYMjMR83dk +sHYf5BA1FxvyDrFspCqjc/wJHx4yGVMR59mzLC52LqGj3n5qiAno8geK+LLNEOfi +c0CTuwjRP+H8C5SzJe98ptfRr5//lpr1kXuYC3fUfugH0mK1lTnj8/FtDw5lhIpj +VMWAtuCeS31HJqcBCF3RiJ7XwzJE+oJKCmhUfzhTA8ykADNkUVkLo4KRel7sFsLz +KuZi2irbWWIQJUoqgQtHB0MGcIfS+pMRKXpITeuUx3BNr2fVUbGAIAEBtHoIppB/ +TuDvB0GHr2qlXov7z1CymlSvw4m6WC31MJixNnI5fkkE/SmnTHnkBVfblLkWU41G +sx2VYVdWf6/wFlthWG82UBEL2KwrlRYaDh8IzTY0ZRBiZtWAXxQgXy0MoHgKaNYs +1+lvK9JKBZP8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0YIrO7o1dfdRUVjzyAfd5LQD +fwIDAQABo0IwQDAdBgNVHQ4EFgQU2XQ65DA9DfcS3H5aBZ8eNJr34RQwDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBADaN +l8xCFWQpN5smLNb7rhVpLGsaGvdftvkHTFnq88nIua7Mui563MD1sC3AO6+fcAUR +ap8lTwEpcOPlDOHqWnzcSbvBHiqB9RZLcpHIojG5qtr8nR/zXUACE/xOHAbKsxSQ +VBcZEhrxH9cMaVr2cXj0lH2RC47skFSOvG+hTKv8dGT9cZr4QQehzZHkPJrgmzI5 +c6sq1WnIeJEmMX3ixzDx/BR4dxIOE/TdFpS/S2d7cFOFyrC78zhNLJA5wA3CXWvp +4uXViI3WLL+rG761KIcSF3Ru/H38j9CHJrAb+7lsq+KePRXBOy5nAliRn+/4Qh8s +t2j1da3Ptfb/EX3C8CSlrdP6oDyp+l3cpaDvRKS+1ujl5BOWF3sGPjLtx7dCvHaj +2GU4Kzg1USEODm8uNBNA4StnDG1KQTAYI1oyVZnJF+A83vbsea0rWBmirSwiGpWO +vpaQXUJXxPkUAzUrHC1RVwinOt4/5Mi0A3PCwSaAuwtCH60NryZy2sy+s6ODWA2C +xR9GUeOcGMyNm43sSet1UNWMKFnKdDTajAshqx7qG+XH/RU+wBeq+yNuJkbL+vmx +cmtpzyKEC2IPrNkZAJSidjzULZrtBJ4tBmIQN1IchXIbJ+XMxjHsN+xjWZsLHXbM +fjKaiJUINlK73nZfdklJrX+9ZSCyycErdhh2n1ax +-----END CERTIFICATE----- + +# Issuer: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Subject: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Label: "Certigna Root CA" +# Serial: 269714418870597844693661054334862075617 +# MD5 Fingerprint: 0e:5c:30:62:27:eb:5b:bc:d7:ae:62:ba:e9:d5:df:77 +# SHA1 Fingerprint: 2d:0d:52:14:ff:9e:ad:99:24:01:74:20:47:6e:6c:85:27:27:f5:43 +# SHA256 Fingerprint: d4:8d:3d:23:ee:db:50:a4:59:e5:51:97:60:1c:27:77:4b:9d:7b:18:c9:4d:5a:05:95:11:a1:02:50:b9:31:68 +-----BEGIN CERTIFICATE----- +MIIGWzCCBEOgAwIBAgIRAMrpG4nxVQMNo+ZBbcTjpuEwDQYJKoZIhvcNAQELBQAw +WjELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCURoaW15b3RpczEcMBoGA1UECwwTMDAw +MiA0ODE0NjMwODEwMDAzNjEZMBcGA1UEAwwQQ2VydGlnbmEgUm9vdCBDQTAeFw0x +MzEwMDEwODMyMjdaFw0zMzEwMDEwODMyMjdaMFoxCzAJBgNVBAYTAkZSMRIwEAYD +VQQKDAlEaGlteW90aXMxHDAaBgNVBAsMEzAwMDIgNDgxNDYzMDgxMDAwMzYxGTAX +BgNVBAMMEENlcnRpZ25hIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQDNGDllGlmx6mQWDoyUJJV8g9PFOSbcDO8WV43X2KyjQn+Cyu3NW9sO +ty3tRQgXstmzy9YXUnIo245Onoq2C/mehJpNdt4iKVzSs9IGPjA5qXSjklYcoW9M +CiBtnyN6tMbaLOQdLNyzKNAT8kxOAkmhVECe5uUFoC2EyP+YbNDrihqECB63aCPu +I9Vwzm1RaRDuoXrC0SIxwoKF0vJVdlB8JXrJhFwLrN1CTivngqIkicuQstDuI7pm +TLtipPlTWmR7fJj6o0ieD5Wupxj0auwuA0Wv8HT4Ks16XdG+RCYyKfHx9WzMfgIh +C59vpD++nVPiz32pLHxYGpfhPTc3GGYo0kDFUYqMwy3OU4gkWGQwFsWq4NYKpkDf +ePb1BHxpE4S80dGnBs8B92jAqFe7OmGtBIyT46388NtEbVncSVmurJqZNjBBe3Yz +IoejwpKGbvlw7q6Hh5UbxHq9MfPU0uWZ/75I7HX1eBYdpnDBfzwboZL7z8g81sWT +Co/1VTp2lc5ZmIoJlXcymoO6LAQ6l73UL77XbJuiyn1tJslV1c/DeVIICZkHJC1k +JWumIWmbat10TWuXekG9qxf5kBdIjzb5LdXF2+6qhUVB+s06RbFo5jZMm5BX7CO5 +hwjCxAnxl4YqKE3idMDaxIzb3+KhF1nOJFl0Mdp//TBt2dzhauH8XwIDAQABo4IB +GjCCARYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FBiHVuBud+4kNTxOc5of1uHieX4rMB8GA1UdIwQYMBaAFBiHVuBud+4kNTxOc5of +1uHieX4rMEQGA1UdIAQ9MDswOQYEVR0gADAxMC8GCCsGAQUFBwIBFiNodHRwczov +L3d3d3cuY2VydGlnbmEuZnIvYXV0b3JpdGVzLzBtBgNVHR8EZjBkMC+gLaArhilo +dHRwOi8vY3JsLmNlcnRpZ25hLmZyL2NlcnRpZ25hcm9vdGNhLmNybDAxoC+gLYYr +aHR0cDovL2NybC5kaGlteW90aXMuY29tL2NlcnRpZ25hcm9vdGNhLmNybDANBgkq +hkiG9w0BAQsFAAOCAgEAlLieT/DjlQgi581oQfccVdV8AOItOoldaDgvUSILSo3L +6btdPrtcPbEo/uRTVRPPoZAbAh1fZkYJMyjhDSSXcNMQH+pkV5a7XdrnxIxPTGRG +HVyH41neQtGbqH6mid2PHMkwgu07nM3A6RngatgCdTer9zQoKJHyBApPNeNgJgH6 +0BGM+RFq7q89w1DTj18zeTyGqHNFkIwgtnJzFyO+B2XleJINugHA64wcZr+shncB +lA2c5uk5jR+mUYyZDDl34bSb+hxnV29qao6pK0xXeXpXIs/NX2NGjVxZOob4Mkdi +o2cNGJHc+6Zr9UhhcyNZjgKnvETq9Emd8VRY+WCv2hikLyhF3HqgiIZd8zvn/yk1 +gPxkQ5Tm4xxvvq0OKmOZK8l+hfZx6AYDlf7ej0gcWtSS6Cvu5zHbugRqh5jnxV/v +faci9wHYTfmJ0A6aBVmknpjZbyvKcL5kwlWj9Omvw5Ip3IgWJJk8jSaYtlu3zM63 +Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh +jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw +3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0= +-----END CERTIFICATE----- diff --git a/pipenv/patched/notpip/_vendor/certifi/core.py b/pipenv/patched/notpip/_vendor/certifi/core.py index eab9d1d1..2d02ea44 100644 --- a/pipenv/patched/notpip/_vendor/certifi/core.py +++ b/pipenv/patched/notpip/_vendor/certifi/core.py @@ -8,14 +8,6 @@ certifi.py This module returns the installation location of cacert.pem. """ import os -import warnings - - -class DeprecatedBundleWarning(DeprecationWarning): - """ - The weak security bundle is being deprecated. Please bother your service - provider to get them to stop using cross-signed roots. - """ def where(): @@ -24,14 +16,5 @@ def where(): return os.path.join(f, 'cacert.pem') -def old_where(): - warnings.warn( - "The weak security bundle has been removed. certifi.old_where() is now an alias " - "of certifi.where(). Please update your code to use certifi.where() instead. " - "certifi.old_where() will be removed in 2018.", - DeprecatedBundleWarning - ) - return where() - if __name__ == '__main__': print(where()) diff --git a/pipenv/patched/notpip/_vendor/colorama/__init__.py b/pipenv/patched/notpip/_vendor/colorama/__init__.py index f4d9ce21..2a3bf471 100644 --- a/pipenv/patched/notpip/_vendor/colorama/__init__.py +++ b/pipenv/patched/notpip/_vendor/colorama/__init__.py @@ -3,5 +3,4 @@ from .initialise import init, deinit, reinit, colorama_text from .ansi import Fore, Back, Style, Cursor from .ansitowin32 import AnsiToWin32 -__version__ = '0.3.9' - +__version__ = '0.4.1' diff --git a/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py b/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py index 1d6e6059..359c92be 100644 --- a/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py +++ b/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py @@ -13,14 +13,6 @@ if windll is not None: winterm = WinTerm() -def is_stream_closed(stream): - return not hasattr(stream, 'closed') or stream.closed - - -def is_a_tty(stream): - return hasattr(stream, 'isatty') and stream.isatty() - - class StreamWrapper(object): ''' Wraps a stream (such as stdout), acting as a transparent proxy for all @@ -36,9 +28,38 @@ class StreamWrapper(object): def __getattr__(self, name): return getattr(self.__wrapped, name) + def __enter__(self, *args, **kwargs): + # special method lookup bypasses __getattr__/__getattribute__, see + # https://stackoverflow.com/questions/12632894/why-doesnt-getattr-work-with-exit + # thus, contextlib magic methods are not proxied via __getattr__ + return self.__wrapped.__enter__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + return self.__wrapped.__exit__(*args, **kwargs) + def write(self, text): self.__convertor.write(text) + def isatty(self): + stream = self.__wrapped + if 'PYCHARM_HOSTED' in os.environ: + if stream is not None and (stream is sys.__stdout__ or stream is sys.__stderr__): + return True + try: + stream_isatty = stream.isatty + except AttributeError: + return False + else: + return stream_isatty() + + @property + def closed(self): + stream = self.__wrapped + try: + return stream.closed + except AttributeError: + return True + class AnsiToWin32(object): ''' @@ -68,12 +89,12 @@ class AnsiToWin32(object): # should we strip ANSI sequences from our output? if strip is None: - strip = conversion_supported or (not is_stream_closed(wrapped) and not is_a_tty(wrapped)) + strip = conversion_supported or (not self.stream.closed and not self.stream.isatty()) self.strip = strip # should we should convert ANSI sequences into win32 calls? if convert is None: - convert = conversion_supported and not is_stream_closed(wrapped) and is_a_tty(wrapped) + convert = conversion_supported and not self.stream.closed and self.stream.isatty() self.convert = convert # dict of ansi codes to win32 functions and parameters @@ -149,7 +170,7 @@ class AnsiToWin32(object): def reset_all(self): if self.convert: self.call_win32('m', (0,)) - elif not self.strip and not is_stream_closed(self.wrapped): + elif not self.strip and not self.stream.closed: self.wrapped.write(Style.RESET_ALL) diff --git a/pipenv/patched/notpip/_vendor/colorama/initialise.py b/pipenv/patched/notpip/_vendor/colorama/initialise.py index 834962a3..430d0668 100644 --- a/pipenv/patched/notpip/_vendor/colorama/initialise.py +++ b/pipenv/patched/notpip/_vendor/colorama/initialise.py @@ -78,5 +78,3 @@ def wrap_stream(stream, convert, strip, autoreset, wrap): if wrapper.should_wrap(): stream = wrapper.stream return stream - - diff --git a/pipenv/patched/notpip/_vendor/colorama/win32.py b/pipenv/patched/notpip/_vendor/colorama/win32.py index 8262e350..c2d83603 100644 --- a/pipenv/patched/notpip/_vendor/colorama/win32.py +++ b/pipenv/patched/notpip/_vendor/colorama/win32.py @@ -89,11 +89,6 @@ else: ] _SetConsoleTitleW.restype = wintypes.BOOL - handles = { - STDOUT: _GetStdHandle(STDOUT), - STDERR: _GetStdHandle(STDERR), - } - def _winapi_test(handle): csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( @@ -101,17 +96,18 @@ else: return bool(success) def winapi_test(): - return any(_winapi_test(h) for h in handles.values()) + return any(_winapi_test(h) for h in + (_GetStdHandle(STDOUT), _GetStdHandle(STDERR))) def GetConsoleScreenBufferInfo(stream_id=STDOUT): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( handle, byref(csbi)) return csbi def SetConsoleTextAttribute(stream_id, attrs): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleTextAttribute(handle, attrs) def SetConsoleCursorPosition(stream_id, position, adjust=True): @@ -129,11 +125,11 @@ else: adjusted_position.Y += sr.Top adjusted_position.X += sr.Left # Resume normal processing - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleCursorPosition(handle, adjusted_position) def FillConsoleOutputCharacter(stream_id, char, length, start): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) char = c_char(char.encode()) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) @@ -144,7 +140,7 @@ else: def FillConsoleOutputAttribute(stream_id, attr, length, start): ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )''' - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) attribute = wintypes.WORD(attr) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) diff --git a/pipenv/patched/notpip/_vendor/colorama/winterm.py b/pipenv/patched/notpip/_vendor/colorama/winterm.py index 60309d3c..0fdb4ec4 100644 --- a/pipenv/patched/notpip/_vendor/colorama/winterm.py +++ b/pipenv/patched/notpip/_vendor/colorama/winterm.py @@ -44,6 +44,7 @@ class WinTerm(object): def reset_all(self, on_stderr=None): self.set_attrs(self._default) self.set_console(attrs=self._default) + self._light = 0 def fore(self, fore=None, light=False, on_stderr=False): if fore is None: @@ -122,12 +123,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = cells_in_screen - cells_before_cursor - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, 0) cells_to_erase = cells_before_cursor elif mode == 2: from_coord = win32.COORD(0, 0) cells_to_erase = cells_in_screen + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly @@ -147,12 +151,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwCursorPosition.X elif mode == 2: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwSize.X + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly diff --git a/pipenv/patched/notpip/_vendor/distlib/__init__.py b/pipenv/patched/notpip/_vendor/distlib/__init__.py index d4aab453..a786b4d3 100644 --- a/pipenv/patched/notpip/_vendor/distlib/__init__.py +++ b/pipenv/patched/notpip/_vendor/distlib/__init__.py @@ -6,7 +6,7 @@ # import logging -__version__ = '0.2.7' +__version__ = '0.2.8' class DistlibException(Exception): pass diff --git a/pipenv/patched/notpip/_vendor/distlib/database.py b/pipenv/patched/notpip/_vendor/distlib/database.py index a19905e2..b13cdac9 100644 --- a/pipenv/patched/notpip/_vendor/distlib/database.py +++ b/pipenv/patched/notpip/_vendor/distlib/database.py @@ -20,7 +20,8 @@ import zipimport from . import DistlibException, resources from .compat import StringIO from .version import get_scheme, UnsupportedVersionError -from .metadata import Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME +from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME, + LEGACY_METADATA_FILENAME) from .util import (parse_requirement, cached_property, parse_name_and_version, read_exports, write_exports, CSVReader, CSVWriter) @@ -132,7 +133,9 @@ class DistributionPath(object): if not r or r.path in seen: continue if self._include_dist and entry.endswith(DISTINFO_EXT): - possible_filenames = [METADATA_FILENAME, WHEEL_METADATA_FILENAME] + possible_filenames = [METADATA_FILENAME, + WHEEL_METADATA_FILENAME, + LEGACY_METADATA_FILENAME] for metadata_filename in possible_filenames: metadata_path = posixpath.join(entry, metadata_filename) pydist = finder.find(metadata_path) diff --git a/pipenv/patched/notpip/_vendor/distlib/locators.py b/pipenv/patched/notpip/_vendor/distlib/locators.py index cb05b184..a7ed9469 100644 --- a/pipenv/patched/notpip/_vendor/distlib/locators.py +++ b/pipenv/patched/notpip/_vendor/distlib/locators.py @@ -255,7 +255,9 @@ class Locator(object): if path.endswith('.whl'): try: wheel = Wheel(path) - if is_compatible(wheel, self.wheel_tags): + if not is_compatible(wheel, self.wheel_tags): + logger.debug('Wheel not compatible: %s', path) + else: if project_name is None: include = True else: @@ -613,6 +615,7 @@ class SimpleScrapingLocator(Locator): # as it is for coordinating our internal threads - the ones created # in _prepare_threads. self._gplock = threading.RLock() + self.platform_check = False # See issue #112 def _prepare_threads(self): """ @@ -658,8 +661,8 @@ class SimpleScrapingLocator(Locator): del self.result return result - platform_dependent = re.compile(r'\b(linux-(i\d86|x86_64|arm\w+)|' - r'win(32|-amd64)|macosx-?\d+)\b', re.I) + platform_dependent = re.compile(r'\b(linux_(i\d86|x86_64|arm\w+)|' + r'win(32|_amd64)|macosx_?\d+)\b', re.I) def _is_platform_dependent(self, url): """ @@ -677,7 +680,7 @@ class SimpleScrapingLocator(Locator): Note that the return value isn't actually used other than as a boolean value. """ - if self._is_platform_dependent(url): + if self.platform_check and self._is_platform_dependent(url): info = None else: info = self.convert_url_to_download_info(url, self.project_name) diff --git a/pipenv/patched/notpip/_vendor/distlib/metadata.py b/pipenv/patched/notpip/_vendor/distlib/metadata.py index 6d6470ff..77eed7f9 100644 --- a/pipenv/patched/notpip/_vendor/distlib/metadata.py +++ b/pipenv/patched/notpip/_vendor/distlib/metadata.py @@ -91,7 +91,9 @@ _426_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', _426_MARKERS = ('Private-Version', 'Provides-Extra', 'Obsoleted-By', 'Setup-Requires-Dist', 'Extension') -_566_FIELDS = _426_FIELDS + ('Description-Content-Type',) +# See issue #106: Sometimes 'Requires' occurs wrongly in the metadata. Include +# it in the tuple literal below to allow it (for now) +_566_FIELDS = _426_FIELDS + ('Description-Content-Type', 'Requires') _566_MARKERS = ('Description-Content-Type',) @@ -377,8 +379,8 @@ class LegacyMetadata(object): value = msg[field] if value is not None and value != 'UNKNOWN': self.set(field, value) - logger.debug('Attempting to set metadata for %s', self) - self.set_metadata_version() + # logger.debug('Attempting to set metadata for %s', self) + # self.set_metadata_version() def write(self, filepath, skip_unknown=False): """Write the metadata fields to filepath.""" @@ -648,6 +650,7 @@ class LegacyMetadata(object): METADATA_FILENAME = 'pydist.json' WHEEL_METADATA_FILENAME = 'metadata.json' +LEGACY_METADATA_FILENAME = 'METADATA' class Metadata(object): diff --git a/pipenv/patched/notpip/_vendor/distlib/scripts.py b/pipenv/patched/notpip/_vendor/distlib/scripts.py index 0b7c3d0b..8e22cb91 100644 --- a/pipenv/patched/notpip/_vendor/distlib/scripts.py +++ b/pipenv/patched/notpip/_vendor/distlib/scripts.py @@ -236,8 +236,10 @@ class ScriptMaker(object): def _write_script(self, names, shebang, script_bytes, filenames, ext): use_launcher = self.add_launchers and self._is_nt linesep = os.linesep.encode('utf-8') + if not shebang.endswith(linesep): + shebang += linesep if not use_launcher: - script_bytes = shebang + linesep + script_bytes + script_bytes = shebang + script_bytes else: # pragma: no cover if ext == 'py': launcher = self._get_launcher('t') @@ -247,7 +249,7 @@ class ScriptMaker(object): with ZipFile(stream, 'w') as zf: zf.writestr('__main__.py', script_bytes) zip_data = stream.getvalue() - script_bytes = launcher + shebang + linesep + zip_data + script_bytes = launcher + shebang + zip_data for name in names: outname = os.path.join(self.target_dir, name) if use_launcher: # pragma: no cover diff --git a/pipenv/patched/notpip/_vendor/distlib/util.py b/pipenv/patched/notpip/_vendor/distlib/util.py index 0b14a93b..9d4bfd3b 100644 --- a/pipenv/patched/notpip/_vendor/distlib/util.py +++ b/pipenv/patched/notpip/_vendor/distlib/util.py @@ -545,16 +545,14 @@ class FileOperator(object): def write_binary_file(self, path, data): self.ensure_dir(os.path.dirname(path)) if not self.dry_run: + if os.path.exists(path): + os.remove(path) with open(path, 'wb') as f: f.write(data) self.record_as_written(path) def write_text_file(self, path, data, encoding): - self.ensure_dir(os.path.dirname(path)) - if not self.dry_run: - with open(path, 'wb') as f: - f.write(data.encode(encoding)) - self.record_as_written(path) + self.write_binary_file(path, data.encode(encoding)) def set_mode(self, bits, mask, files): if os.name == 'posix' or (os.name == 'java' and os._name == 'posix'): @@ -582,7 +580,7 @@ class FileOperator(object): if self.record: self.dirs_created.add(path) - def byte_compile(self, path, optimize=False, force=False, prefix=None): + def byte_compile(self, path, optimize=False, force=False, prefix=None, hashed_invalidation=False): dpath = cache_from_source(path, not optimize) logger.info('Byte-compiling %s to %s', path, dpath) if not self.dry_run: @@ -592,7 +590,10 @@ class FileOperator(object): else: assert path.startswith(prefix) diagpath = path[len(prefix):] - py_compile.compile(path, dpath, diagpath, True) # raise error + compile_kwargs = {} + if hashed_invalidation and hasattr(py_compile, 'PycInvalidationMode'): + compile_kwargs['invalidation_mode'] = py_compile.PycInvalidationMode.CHECKED_HASH + py_compile.compile(path, dpath, diagpath, True, **compile_kwargs) # raise error self.record_as_written(dpath) return dpath diff --git a/pipenv/patched/notpip/_vendor/distlib/wheel.py b/pipenv/patched/notpip/_vendor/distlib/wheel.py index 77372235..b04bfaef 100644 --- a/pipenv/patched/notpip/_vendor/distlib/wheel.py +++ b/pipenv/patched/notpip/_vendor/distlib/wheel.py @@ -442,7 +442,9 @@ class Wheel(object): This can be used to issue any warnings to raise any exceptions. If kwarg ``lib_only`` is True, only the purelib/platlib files are installed, and the headers, scripts, data and dist-info metadata are - not written. + not written. If kwarg ``bytecode_hashed_invalidation`` is True, written + bytecode will try to use file-hash based invalidation (PEP-552) on + supported interpreter versions (CPython 2.7+). The return value is a :class:`InstalledDistribution` instance unless ``options.lib_only`` is True, in which case the return value is ``None``. @@ -451,6 +453,7 @@ class Wheel(object): dry_run = maker.dry_run warner = kwargs.get('warner') lib_only = kwargs.get('lib_only', False) + bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', False) pathname = os.path.join(self.dirname, self.filename) name_ver = '%s-%s' % (self.name, self.version) @@ -557,7 +560,8 @@ class Wheel(object): '%s' % outfile) if bc and outfile.endswith('.py'): try: - pyc = fileop.byte_compile(outfile) + pyc = fileop.byte_compile(outfile, + hashed_invalidation=bc_hashed_invalidation) outfiles.append(pyc) except Exception: # Don't give up if byte-compilation fails, diff --git a/pipenv/patched/notpip/_vendor/idna/core.py b/pipenv/patched/notpip/_vendor/idna/core.py index 090c2c18..104624ad 100644 --- a/pipenv/patched/notpip/_vendor/idna/core.py +++ b/pipenv/patched/notpip/_vendor/idna/core.py @@ -267,10 +267,7 @@ def alabel(label): try: label = label.encode('ascii') - try: - ulabel(label) - except IDNAError: - raise IDNAError('The label {0} is not a valid A-label'.format(label)) + ulabel(label) if not valid_label_length(label): raise IDNAError('Label too long') return label diff --git a/pipenv/patched/notpip/_vendor/idna/idnadata.py b/pipenv/patched/notpip/_vendor/idna/idnadata.py index 17974e23..a80c959d 100644 --- a/pipenv/patched/notpip/_vendor/idna/idnadata.py +++ b/pipenv/patched/notpip/_vendor/idna/idnadata.py @@ -1,6 +1,6 @@ # This file is automatically generated by tools/idna-data -__version__ = "10.0.0" +__version__ = "11.0.0" scripts = { 'Greek': ( 0x37000000374, @@ -49,7 +49,7 @@ scripts = { 0x30210000302a, 0x30380000303c, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xf9000000fa6e, 0xfa700000fada, 0x200000002a6d7, @@ -62,7 +62,7 @@ scripts = { 'Hebrew': ( 0x591000005c8, 0x5d0000005eb, - 0x5f0000005f5, + 0x5ef000005f5, 0xfb1d0000fb37, 0xfb380000fb3d, 0xfb3e0000fb3f, @@ -248,6 +248,7 @@ joining_types = { 0x6fb: 68, 0x6fc: 68, 0x6ff: 68, + 0x70f: 84, 0x710: 82, 0x712: 68, 0x713: 68, @@ -522,6 +523,7 @@ joining_types = { 0x1875: 68, 0x1876: 68, 0x1877: 68, + 0x1878: 68, 0x1880: 85, 0x1881: 85, 0x1882: 85, @@ -690,6 +692,70 @@ joining_types = { 0x10bad: 68, 0x10bae: 68, 0x10baf: 85, + 0x10d00: 76, + 0x10d01: 68, + 0x10d02: 68, + 0x10d03: 68, + 0x10d04: 68, + 0x10d05: 68, + 0x10d06: 68, + 0x10d07: 68, + 0x10d08: 68, + 0x10d09: 68, + 0x10d0a: 68, + 0x10d0b: 68, + 0x10d0c: 68, + 0x10d0d: 68, + 0x10d0e: 68, + 0x10d0f: 68, + 0x10d10: 68, + 0x10d11: 68, + 0x10d12: 68, + 0x10d13: 68, + 0x10d14: 68, + 0x10d15: 68, + 0x10d16: 68, + 0x10d17: 68, + 0x10d18: 68, + 0x10d19: 68, + 0x10d1a: 68, + 0x10d1b: 68, + 0x10d1c: 68, + 0x10d1d: 68, + 0x10d1e: 68, + 0x10d1f: 68, + 0x10d20: 68, + 0x10d21: 68, + 0x10d22: 82, + 0x10d23: 68, + 0x10f30: 68, + 0x10f31: 68, + 0x10f32: 68, + 0x10f33: 82, + 0x10f34: 68, + 0x10f35: 68, + 0x10f36: 68, + 0x10f37: 68, + 0x10f38: 68, + 0x10f39: 68, + 0x10f3a: 68, + 0x10f3b: 68, + 0x10f3c: 68, + 0x10f3d: 68, + 0x10f3e: 68, + 0x10f3f: 68, + 0x10f40: 68, + 0x10f41: 68, + 0x10f42: 68, + 0x10f43: 68, + 0x10f44: 68, + 0x10f45: 85, + 0x10f51: 68, + 0x10f52: 68, + 0x10f53: 68, + 0x10f54: 82, + 0x110bd: 85, + 0x110cd: 85, 0x1e900: 68, 0x1e901: 68, 0x1e902: 68, @@ -1034,14 +1100,15 @@ codepoint_classes = { 0x52d0000052e, 0x52f00000530, 0x5590000055a, - 0x56100000587, + 0x56000000587, + 0x58800000589, 0x591000005be, 0x5bf000005c0, 0x5c1000005c3, 0x5c4000005c6, 0x5c7000005c8, 0x5d0000005eb, - 0x5f0000005f3, + 0x5ef000005f3, 0x6100000061b, 0x62000000640, 0x64100000660, @@ -1054,12 +1121,13 @@ codepoint_classes = { 0x7100000074b, 0x74d000007b2, 0x7c0000007f6, + 0x7fd000007fe, 0x8000000082e, 0x8400000085c, 0x8600000086b, 0x8a0000008b5, 0x8b6000008be, - 0x8d4000008e2, + 0x8d3000008e2, 0x8e300000958, 0x96000000964, 0x96600000970, @@ -1077,6 +1145,7 @@ codepoint_classes = { 0x9e0000009e4, 0x9e6000009f2, 0x9fc000009fd, + 0x9fe000009ff, 0xa0100000a04, 0xa0500000a0b, 0xa0f00000a11, @@ -1136,8 +1205,7 @@ codepoint_classes = { 0xbd000000bd1, 0xbd700000bd8, 0xbe600000bf0, - 0xc0000000c04, - 0xc0500000c0d, + 0xc0000000c0d, 0xc0e00000c11, 0xc1200000c29, 0xc2a00000c3a, @@ -1276,7 +1344,7 @@ codepoint_classes = { 0x17dc000017de, 0x17e0000017ea, 0x18100000181a, - 0x182000001878, + 0x182000001879, 0x1880000018ab, 0x18b0000018f6, 0x19000000191f, @@ -1544,11 +1612,11 @@ codepoint_classes = { 0x309d0000309f, 0x30a1000030fb, 0x30fc000030ff, - 0x31050000312f, + 0x310500003130, 0x31a0000031bb, 0x31f000003200, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xa0000000a48d, 0xa4d00000a4fe, 0xa5000000a60d, @@ -1655,8 +1723,10 @@ codepoint_classes = { 0xa7a50000a7a6, 0xa7a70000a7a8, 0xa7a90000a7aa, + 0xa7af0000a7b0, 0xa7b50000a7b6, 0xa7b70000a7b8, + 0xa7b90000a7ba, 0xa7f70000a7f8, 0xa7fa0000a828, 0xa8400000a874, @@ -1664,8 +1734,7 @@ codepoint_classes = { 0xa8d00000a8da, 0xa8e00000a8f8, 0xa8fb0000a8fc, - 0xa8fd0000a8fe, - 0xa9000000a92e, + 0xa8fd0000a92e, 0xa9300000a954, 0xa9800000a9c1, 0xa9cf0000a9da, @@ -1743,7 +1812,7 @@ codepoint_classes = { 0x10a0500010a07, 0x10a0c00010a14, 0x10a1500010a18, - 0x10a1900010a34, + 0x10a1900010a36, 0x10a3800010a3b, 0x10a3f00010a40, 0x10a6000010a7d, @@ -1756,6 +1825,11 @@ codepoint_classes = { 0x10b8000010b92, 0x10c0000010c49, 0x10cc000010cf3, + 0x10d0000010d28, + 0x10d3000010d3a, + 0x10f0000010f1d, + 0x10f2700010f28, + 0x10f3000010f51, 0x1100000011047, 0x1106600011070, 0x1107f000110bb, @@ -1763,10 +1837,11 @@ codepoint_classes = { 0x110f0000110fa, 0x1110000011135, 0x1113600011140, + 0x1114400011147, 0x1115000011174, 0x1117600011177, 0x11180000111c5, - 0x111ca000111cd, + 0x111c9000111cd, 0x111d0000111db, 0x111dc000111dd, 0x1120000011212, @@ -1786,7 +1861,7 @@ codepoint_classes = { 0x1132a00011331, 0x1133200011334, 0x113350001133a, - 0x1133c00011345, + 0x1133b00011345, 0x1134700011349, 0x1134b0001134e, 0x1135000011351, @@ -1796,6 +1871,7 @@ codepoint_classes = { 0x1137000011375, 0x114000001144b, 0x114500001145a, + 0x1145e0001145f, 0x11480000114c6, 0x114c7000114c8, 0x114d0000114da, @@ -1807,15 +1883,17 @@ codepoint_classes = { 0x116500001165a, 0x11680000116b8, 0x116c0000116ca, - 0x117000001171a, + 0x117000001171b, 0x1171d0001172c, 0x117300001173a, + 0x118000001183b, 0x118c0000118ea, 0x118ff00011900, 0x11a0000011a3f, 0x11a4700011a48, 0x11a5000011a84, 0x11a8600011a9a, + 0x11a9d00011a9e, 0x11ac000011af9, 0x11c0000011c09, 0x11c0a00011c37, @@ -1831,6 +1909,13 @@ codepoint_classes = { 0x11d3c00011d3e, 0x11d3f00011d48, 0x11d5000011d5a, + 0x11d6000011d66, + 0x11d6700011d69, + 0x11d6a00011d8f, + 0x11d9000011d92, + 0x11d9300011d99, + 0x11da000011daa, + 0x11ee000011ef7, 0x120000001239a, 0x1248000012544, 0x130000001342f, @@ -1845,11 +1930,12 @@ codepoint_classes = { 0x16b5000016b5a, 0x16b6300016b78, 0x16b7d00016b90, + 0x16e6000016e80, 0x16f0000016f45, 0x16f5000016f7f, 0x16f8f00016fa0, 0x16fe000016fe2, - 0x17000000187ed, + 0x17000000187f2, 0x1880000018af3, 0x1b0000001b11f, 0x1b1700001b2fc, diff --git a/pipenv/patched/notpip/_vendor/idna/package_data.py b/pipenv/patched/notpip/_vendor/idna/package_data.py index 39c192ba..257e8989 100644 --- a/pipenv/patched/notpip/_vendor/idna/package_data.py +++ b/pipenv/patched/notpip/_vendor/idna/package_data.py @@ -1,2 +1,2 @@ -__version__ = '2.7' +__version__ = '2.8' diff --git a/pipenv/patched/notpip/_vendor/idna/uts46data.py b/pipenv/patched/notpip/_vendor/idna/uts46data.py index 79731cb9..a68ed4c0 100644 --- a/pipenv/patched/notpip/_vendor/idna/uts46data.py +++ b/pipenv/patched/notpip/_vendor/idna/uts46data.py @@ -4,7 +4,7 @@ """IDNA Mapping Table from UTS46.""" -__version__ = "10.0.0" +__version__ = "11.0.0" def _seg_0(): return [ (0x0, '3'), @@ -1029,11 +1029,8 @@ def _seg_9(): (0x556, 'M', u'ֆ'), (0x557, 'X'), (0x559, 'V'), - (0x560, 'X'), - (0x561, 'V'), (0x587, 'M', u'եւ'), - (0x588, 'X'), - (0x589, 'V'), + (0x588, 'V'), (0x58B, 'X'), (0x58D, 'V'), (0x590, 'X'), @@ -1041,15 +1038,15 @@ def _seg_9(): (0x5C8, 'X'), (0x5D0, 'V'), (0x5EB, 'X'), - (0x5F0, 'V'), + (0x5EF, 'V'), (0x5F5, 'X'), + (0x606, 'V'), + (0x61C, 'X'), + (0x61E, 'V'), ] def _seg_10(): return [ - (0x606, 'V'), - (0x61C, 'X'), - (0x61E, 'V'), (0x675, 'M', u'اٴ'), (0x676, 'M', u'وٴ'), (0x677, 'M', u'ۇٴ'), @@ -1064,7 +1061,7 @@ def _seg_10(): (0x7B2, 'X'), (0x7C0, 'V'), (0x7FB, 'X'), - (0x800, 'V'), + (0x7FD, 'V'), (0x82E, 'X'), (0x830, 'V'), (0x83F, 'X'), @@ -1078,7 +1075,7 @@ def _seg_10(): (0x8B5, 'X'), (0x8B6, 'V'), (0x8BE, 'X'), - (0x8D4, 'V'), + (0x8D3, 'V'), (0x8E2, 'X'), (0x8E3, 'V'), (0x958, 'M', u'क़'), @@ -1118,7 +1115,7 @@ def _seg_10(): (0x9E0, 'V'), (0x9E4, 'X'), (0x9E6, 'V'), - (0x9FE, 'X'), + (0x9FF, 'X'), (0xA01, 'V'), (0xA04, 'X'), (0xA05, 'V'), @@ -1147,19 +1144,19 @@ def _seg_10(): (0xA4E, 'X'), (0xA51, 'V'), (0xA52, 'X'), + (0xA59, 'M', u'ਖ਼'), + (0xA5A, 'M', u'ਗ਼'), + (0xA5B, 'M', u'ਜ਼'), ] def _seg_11(): return [ - (0xA59, 'M', u'ਖ਼'), - (0xA5A, 'M', u'ਗ਼'), - (0xA5B, 'M', u'ਜ਼'), (0xA5C, 'V'), (0xA5D, 'X'), (0xA5E, 'M', u'ਫ਼'), (0xA5F, 'X'), (0xA66, 'V'), - (0xA76, 'X'), + (0xA77, 'X'), (0xA81, 'V'), (0xA84, 'X'), (0xA85, 'V'), @@ -1250,16 +1247,14 @@ def _seg_11(): (0xBE6, 'V'), (0xBFB, 'X'), (0xC00, 'V'), - (0xC04, 'X'), - ] - -def _seg_12(): - return [ - (0xC05, 'V'), (0xC0D, 'X'), (0xC0E, 'V'), (0xC11, 'X'), (0xC12, 'V'), + ] + +def _seg_12(): + return [ (0xC29, 'X'), (0xC2A, 'V'), (0xC3A, 'X'), @@ -1278,8 +1273,6 @@ def _seg_12(): (0xC66, 'V'), (0xC70, 'X'), (0xC78, 'V'), - (0xC84, 'X'), - (0xC85, 'V'), (0xC8D, 'X'), (0xC8E, 'V'), (0xC91, 'X'), @@ -1355,10 +1348,6 @@ def _seg_12(): (0xE83, 'X'), (0xE84, 'V'), (0xE85, 'X'), - ] - -def _seg_13(): - return [ (0xE87, 'V'), (0xE89, 'X'), (0xE8A, 'V'), @@ -1366,6 +1355,10 @@ def _seg_13(): (0xE8D, 'V'), (0xE8E, 'X'), (0xE94, 'V'), + ] + +def _seg_13(): + return [ (0xE98, 'X'), (0xE99, 'V'), (0xEA0, 'X'), @@ -1459,10 +1452,6 @@ def _seg_13(): (0x124E, 'X'), (0x1250, 'V'), (0x1257, 'X'), - ] - -def _seg_14(): - return [ (0x1258, 'V'), (0x1259, 'X'), (0x125A, 'V'), @@ -1470,6 +1459,10 @@ def _seg_14(): (0x1260, 'V'), (0x1289, 'X'), (0x128A, 'V'), + ] + +def _seg_14(): + return [ (0x128E, 'X'), (0x1290, 'V'), (0x12B1, 'X'), @@ -1538,7 +1531,7 @@ def _seg_14(): (0x1810, 'V'), (0x181A, 'X'), (0x1820, 'V'), - (0x1878, 'X'), + (0x1879, 'X'), (0x1880, 'V'), (0x18AB, 'X'), (0x18B0, 'V'), @@ -1563,10 +1556,6 @@ def _seg_14(): (0x19DB, 'X'), (0x19DE, 'V'), (0x1A1C, 'X'), - ] - -def _seg_15(): - return [ (0x1A1E, 'V'), (0x1A5F, 'X'), (0x1A60, 'V'), @@ -1574,6 +1563,10 @@ def _seg_15(): (0x1A7F, 'V'), (0x1A8A, 'X'), (0x1A90, 'V'), + ] + +def _seg_15(): + return [ (0x1A9A, 'X'), (0x1AA0, 'V'), (0x1AAE, 'X'), @@ -1667,10 +1660,6 @@ def _seg_15(): (0x1D68, 'M', u'ρ'), (0x1D69, 'M', u'φ'), (0x1D6A, 'M', u'χ'), - ] - -def _seg_16(): - return [ (0x1D6B, 'V'), (0x1D78, 'M', u'н'), (0x1D79, 'V'), @@ -1678,6 +1667,10 @@ def _seg_16(): (0x1D9C, 'M', u'c'), (0x1D9D, 'M', u'ɕ'), (0x1D9E, 'M', u'ð'), + ] + +def _seg_16(): + return [ (0x1D9F, 'M', u'ɜ'), (0x1DA0, 'M', u'f'), (0x1DA1, 'M', u'ɟ'), @@ -1771,10 +1764,6 @@ def _seg_16(): (0x1E36, 'M', u'ḷ'), (0x1E37, 'V'), (0x1E38, 'M', u'ḹ'), - ] - -def _seg_17(): - return [ (0x1E39, 'V'), (0x1E3A, 'M', u'ḻ'), (0x1E3B, 'V'), @@ -1782,6 +1771,10 @@ def _seg_17(): (0x1E3D, 'V'), (0x1E3E, 'M', u'ḿ'), (0x1E3F, 'V'), + ] + +def _seg_17(): + return [ (0x1E40, 'M', u'ṁ'), (0x1E41, 'V'), (0x1E42, 'M', u'ṃ'), @@ -1875,10 +1868,6 @@ def _seg_17(): (0x1E9F, 'V'), (0x1EA0, 'M', u'ạ'), (0x1EA1, 'V'), - ] - -def _seg_18(): - return [ (0x1EA2, 'M', u'ả'), (0x1EA3, 'V'), (0x1EA4, 'M', u'ấ'), @@ -1886,6 +1875,10 @@ def _seg_18(): (0x1EA6, 'M', u'ầ'), (0x1EA7, 'V'), (0x1EA8, 'M', u'ẩ'), + ] + +def _seg_18(): + return [ (0x1EA9, 'V'), (0x1EAA, 'M', u'ẫ'), (0x1EAB, 'V'), @@ -1979,10 +1972,6 @@ def _seg_18(): (0x1F0B, 'M', u'ἃ'), (0x1F0C, 'M', u'ἄ'), (0x1F0D, 'M', u'ἅ'), - ] - -def _seg_19(): - return [ (0x1F0E, 'M', u'ἆ'), (0x1F0F, 'M', u'ἇ'), (0x1F10, 'V'), @@ -1990,6 +1979,10 @@ def _seg_19(): (0x1F18, 'M', u'ἐ'), (0x1F19, 'M', u'ἑ'), (0x1F1A, 'M', u'ἒ'), + ] + +def _seg_19(): + return [ (0x1F1B, 'M', u'ἓ'), (0x1F1C, 'M', u'ἔ'), (0x1F1D, 'M', u'ἕ'), @@ -2083,10 +2076,6 @@ def _seg_19(): (0x1F9A, 'M', u'ἢι'), (0x1F9B, 'M', u'ἣι'), (0x1F9C, 'M', u'ἤι'), - ] - -def _seg_20(): - return [ (0x1F9D, 'M', u'ἥι'), (0x1F9E, 'M', u'ἦι'), (0x1F9F, 'M', u'ἧι'), @@ -2094,6 +2083,10 @@ def _seg_20(): (0x1FA1, 'M', u'ὡι'), (0x1FA2, 'M', u'ὢι'), (0x1FA3, 'M', u'ὣι'), + ] + +def _seg_20(): + return [ (0x1FA4, 'M', u'ὤι'), (0x1FA5, 'M', u'ὥι'), (0x1FA6, 'M', u'ὦι'), @@ -2187,10 +2180,6 @@ def _seg_20(): (0x2024, 'X'), (0x2027, 'V'), (0x2028, 'X'), - ] - -def _seg_21(): - return [ (0x202F, '3', u' '), (0x2030, 'V'), (0x2033, 'M', u'′′'), @@ -2198,6 +2187,10 @@ def _seg_21(): (0x2035, 'V'), (0x2036, 'M', u'‵‵'), (0x2037, 'M', u'‵‵‵'), + ] + +def _seg_21(): + return [ (0x2038, 'V'), (0x203C, '3', u'!!'), (0x203D, 'V'), @@ -2291,10 +2284,6 @@ def _seg_21(): (0x2120, 'M', u'sm'), (0x2121, 'M', u'tel'), (0x2122, 'M', u'tm'), - ] - -def _seg_22(): - return [ (0x2123, 'V'), (0x2124, 'M', u'z'), (0x2125, 'V'), @@ -2302,6 +2291,10 @@ def _seg_22(): (0x2127, 'V'), (0x2128, 'M', u'z'), (0x2129, 'V'), + ] + +def _seg_22(): + return [ (0x212A, 'M', u'k'), (0x212B, 'M', u'å'), (0x212C, 'M', u'b'), @@ -2395,10 +2388,6 @@ def _seg_22(): (0x226E, '3'), (0x2270, 'V'), (0x2329, 'M', u'〈'), - ] - -def _seg_23(): - return [ (0x232A, 'M', u'〉'), (0x232B, 'V'), (0x2427, 'X'), @@ -2406,6 +2395,10 @@ def _seg_23(): (0x244B, 'X'), (0x2460, 'M', u'1'), (0x2461, 'M', u'2'), + ] + +def _seg_23(): + return [ (0x2462, 'M', u'3'), (0x2463, 'M', u'4'), (0x2464, 'M', u'5'), @@ -2499,10 +2492,6 @@ def _seg_23(): (0x24CF, 'M', u'z'), (0x24D0, 'M', u'a'), (0x24D1, 'M', u'b'), - ] - -def _seg_24(): - return [ (0x24D2, 'M', u'c'), (0x24D3, 'M', u'd'), (0x24D4, 'M', u'e'), @@ -2510,6 +2499,10 @@ def _seg_24(): (0x24D6, 'M', u'g'), (0x24D7, 'M', u'h'), (0x24D8, 'M', u'i'), + ] + +def _seg_24(): + return [ (0x24D9, 'M', u'j'), (0x24DA, 'M', u'k'), (0x24DB, 'M', u'l'), @@ -2541,13 +2534,9 @@ def _seg_24(): (0x2B76, 'V'), (0x2B96, 'X'), (0x2B98, 'V'), - (0x2BBA, 'X'), - (0x2BBD, 'V'), (0x2BC9, 'X'), (0x2BCA, 'V'), - (0x2BD3, 'X'), - (0x2BEC, 'V'), - (0x2BF0, 'X'), + (0x2BFF, 'X'), (0x2C00, 'M', u'ⰰ'), (0x2C01, 'M', u'ⰱ'), (0x2C02, 'M', u'ⰲ'), @@ -2603,10 +2592,6 @@ def _seg_24(): (0x2C62, 'M', u'ɫ'), (0x2C63, 'M', u'ᵽ'), (0x2C64, 'M', u'ɽ'), - ] - -def _seg_25(): - return [ (0x2C65, 'V'), (0x2C67, 'M', u'ⱨ'), (0x2C68, 'V'), @@ -2618,6 +2603,10 @@ def _seg_25(): (0x2C6E, 'M', u'ɱ'), (0x2C6F, 'M', u'ɐ'), (0x2C70, 'M', u'ɒ'), + ] + +def _seg_25(): + return [ (0x2C71, 'V'), (0x2C72, 'M', u'ⱳ'), (0x2C73, 'V'), @@ -2707,10 +2696,6 @@ def _seg_25(): (0x2CCD, 'V'), (0x2CCE, 'M', u'ⳏ'), (0x2CCF, 'V'), - ] - -def _seg_26(): - return [ (0x2CD0, 'M', u'ⳑ'), (0x2CD1, 'V'), (0x2CD2, 'M', u'ⳓ'), @@ -2722,6 +2707,10 @@ def _seg_26(): (0x2CD8, 'M', u'ⳙ'), (0x2CD9, 'V'), (0x2CDA, 'M', u'ⳛ'), + ] + +def _seg_26(): + return [ (0x2CDB, 'V'), (0x2CDC, 'M', u'ⳝ'), (0x2CDD, 'V'), @@ -2768,7 +2757,7 @@ def _seg_26(): (0x2DD8, 'V'), (0x2DDF, 'X'), (0x2DE0, 'V'), - (0x2E4A, 'X'), + (0x2E4F, 'X'), (0x2E80, 'V'), (0x2E9A, 'X'), (0x2E9B, 'V'), @@ -2811,10 +2800,6 @@ def _seg_26(): (0x2F20, 'M', u'士'), (0x2F21, 'M', u'夂'), (0x2F22, 'M', u'夊'), - ] - -def _seg_27(): - return [ (0x2F23, 'M', u'夕'), (0x2F24, 'M', u'大'), (0x2F25, 'M', u'女'), @@ -2826,6 +2811,10 @@ def _seg_27(): (0x2F2B, 'M', u'尸'), (0x2F2C, 'M', u'屮'), (0x2F2D, 'M', u'山'), + ] + +def _seg_27(): + return [ (0x2F2E, 'M', u'巛'), (0x2F2F, 'M', u'工'), (0x2F30, 'M', u'己'), @@ -2915,10 +2904,6 @@ def _seg_27(): (0x2F84, 'M', u'至'), (0x2F85, 'M', u'臼'), (0x2F86, 'M', u'舌'), - ] - -def _seg_28(): - return [ (0x2F87, 'M', u'舛'), (0x2F88, 'M', u'舟'), (0x2F89, 'M', u'艮'), @@ -2930,6 +2915,10 @@ def _seg_28(): (0x2F8F, 'M', u'行'), (0x2F90, 'M', u'衣'), (0x2F91, 'M', u'襾'), + ] + +def _seg_28(): + return [ (0x2F92, 'M', u'見'), (0x2F93, 'M', u'角'), (0x2F94, 'M', u'言'), @@ -3019,13 +3008,9 @@ def _seg_28(): (0x309F, 'M', u'より'), (0x30A0, 'V'), (0x30FF, 'M', u'コト'), - ] - -def _seg_29(): - return [ (0x3100, 'X'), (0x3105, 'V'), - (0x312F, 'X'), + (0x3130, 'X'), (0x3131, 'M', u'ᄀ'), (0x3132, 'M', u'ᄁ'), (0x3133, 'M', u'ᆪ'), @@ -3034,6 +3019,10 @@ def _seg_29(): (0x3136, 'M', u'ᆭ'), (0x3137, 'M', u'ᄃ'), (0x3138, 'M', u'ᄄ'), + ] + +def _seg_29(): + return [ (0x3139, 'M', u'ᄅ'), (0x313A, 'M', u'ᆰ'), (0x313B, 'M', u'ᆱ'), @@ -3123,10 +3112,6 @@ def _seg_29(): (0x318F, 'X'), (0x3190, 'V'), (0x3192, 'M', u'一'), - ] - -def _seg_30(): - return [ (0x3193, 'M', u'二'), (0x3194, 'M', u'三'), (0x3195, 'M', u'四'), @@ -3138,6 +3123,10 @@ def _seg_30(): (0x319B, 'M', u'丙'), (0x319C, 'M', u'丁'), (0x319D, 'M', u'天'), + ] + +def _seg_30(): + return [ (0x319E, 'M', u'地'), (0x319F, 'M', u'人'), (0x31A0, 'V'), @@ -3227,10 +3216,6 @@ def _seg_30(): (0x3256, 'M', u'26'), (0x3257, 'M', u'27'), (0x3258, 'M', u'28'), - ] - -def _seg_31(): - return [ (0x3259, 'M', u'29'), (0x325A, 'M', u'30'), (0x325B, 'M', u'31'), @@ -3242,6 +3227,10 @@ def _seg_31(): (0x3261, 'M', u'ᄂ'), (0x3262, 'M', u'ᄃ'), (0x3263, 'M', u'ᄅ'), + ] + +def _seg_31(): + return [ (0x3264, 'M', u'ᄆ'), (0x3265, 'M', u'ᄇ'), (0x3266, 'M', u'ᄉ'), @@ -3331,10 +3320,6 @@ def _seg_31(): (0x32BA, 'M', u'45'), (0x32BB, 'M', u'46'), (0x32BC, 'M', u'47'), - ] - -def _seg_32(): - return [ (0x32BD, 'M', u'48'), (0x32BE, 'M', u'49'), (0x32BF, 'M', u'50'), @@ -3346,6 +3331,10 @@ def _seg_32(): (0x32C5, 'M', u'6月'), (0x32C6, 'M', u'7月'), (0x32C7, 'M', u'8月'), + ] + +def _seg_32(): + return [ (0x32C8, 'M', u'9月'), (0x32C9, 'M', u'10月'), (0x32CA, 'M', u'11月'), @@ -3435,10 +3424,6 @@ def _seg_32(): (0x331E, 'M', u'コーポ'), (0x331F, 'M', u'サイクル'), (0x3320, 'M', u'サンチーム'), - ] - -def _seg_33(): - return [ (0x3321, 'M', u'シリング'), (0x3322, 'M', u'センチ'), (0x3323, 'M', u'セント'), @@ -3450,6 +3435,10 @@ def _seg_33(): (0x3329, 'M', u'ノット'), (0x332A, 'M', u'ハイツ'), (0x332B, 'M', u'パーセント'), + ] + +def _seg_33(): + return [ (0x332C, 'M', u'パーツ'), (0x332D, 'M', u'バーレル'), (0x332E, 'M', u'ピアストル'), @@ -3539,10 +3528,6 @@ def _seg_33(): (0x3382, 'M', u'μa'), (0x3383, 'M', u'ma'), (0x3384, 'M', u'ka'), - ] - -def _seg_34(): - return [ (0x3385, 'M', u'kb'), (0x3386, 'M', u'mb'), (0x3387, 'M', u'gb'), @@ -3554,6 +3539,10 @@ def _seg_34(): (0x338D, 'M', u'μg'), (0x338E, 'M', u'mg'), (0x338F, 'M', u'kg'), + ] + +def _seg_34(): + return [ (0x3390, 'M', u'hz'), (0x3391, 'M', u'khz'), (0x3392, 'M', u'mhz'), @@ -3643,10 +3632,6 @@ def _seg_34(): (0x33E6, 'M', u'7日'), (0x33E7, 'M', u'8日'), (0x33E8, 'M', u'9日'), - ] - -def _seg_35(): - return [ (0x33E9, 'M', u'10日'), (0x33EA, 'M', u'11日'), (0x33EB, 'M', u'12日'), @@ -3658,6 +3643,10 @@ def _seg_35(): (0x33F1, 'M', u'18日'), (0x33F2, 'M', u'19日'), (0x33F3, 'M', u'20日'), + ] + +def _seg_35(): + return [ (0x33F4, 'M', u'21日'), (0x33F5, 'M', u'22日'), (0x33F6, 'M', u'23日'), @@ -3673,7 +3662,7 @@ def _seg_35(): (0x3400, 'V'), (0x4DB6, 'X'), (0x4DC0, 'V'), - (0x9FEB, 'X'), + (0x9FF0, 'X'), (0xA000, 'V'), (0xA48D, 'X'), (0xA490, 'V'), @@ -3747,10 +3736,6 @@ def _seg_35(): (0xA692, 'M', u'ꚓ'), (0xA693, 'V'), (0xA694, 'M', u'ꚕ'), - ] - -def _seg_36(): - return [ (0xA695, 'V'), (0xA696, 'M', u'ꚗ'), (0xA697, 'V'), @@ -3762,6 +3747,10 @@ def _seg_36(): (0xA69D, 'M', u'ь'), (0xA69E, 'V'), (0xA6F8, 'X'), + ] + +def _seg_36(): + return [ (0xA700, 'V'), (0xA722, 'M', u'ꜣ'), (0xA723, 'V'), @@ -3851,10 +3840,6 @@ def _seg_36(): (0xA780, 'M', u'ꞁ'), (0xA781, 'V'), (0xA782, 'M', u'ꞃ'), - ] - -def _seg_37(): - return [ (0xA783, 'V'), (0xA784, 'M', u'ꞅ'), (0xA785, 'V'), @@ -3866,6 +3851,10 @@ def _seg_37(): (0xA78E, 'V'), (0xA790, 'M', u'ꞑ'), (0xA791, 'V'), + ] + +def _seg_37(): + return [ (0xA792, 'M', u'ꞓ'), (0xA793, 'V'), (0xA796, 'M', u'ꞗ'), @@ -3893,7 +3882,7 @@ def _seg_37(): (0xA7AC, 'M', u'ɡ'), (0xA7AD, 'M', u'ɬ'), (0xA7AE, 'M', u'ɪ'), - (0xA7AF, 'X'), + (0xA7AF, 'V'), (0xA7B0, 'M', u'ʞ'), (0xA7B1, 'M', u'ʇ'), (0xA7B2, 'M', u'ʝ'), @@ -3903,6 +3892,8 @@ def _seg_37(): (0xA7B6, 'M', u'ꞷ'), (0xA7B7, 'V'), (0xA7B8, 'X'), + (0xA7B9, 'V'), + (0xA7BA, 'X'), (0xA7F7, 'V'), (0xA7F8, 'M', u'ħ'), (0xA7F9, 'M', u'œ'), @@ -3917,8 +3908,6 @@ def _seg_37(): (0xA8CE, 'V'), (0xA8DA, 'X'), (0xA8E0, 'V'), - (0xA8FE, 'X'), - (0xA900, 'V'), (0xA954, 'X'), (0xA95F, 'V'), (0xA97D, 'X'), @@ -3955,10 +3944,6 @@ def _seg_37(): (0xAB5F, 'M', u'ꭒ'), (0xAB60, 'V'), (0xAB66, 'X'), - ] - -def _seg_38(): - return [ (0xAB70, 'M', u'Ꭰ'), (0xAB71, 'M', u'Ꭱ'), (0xAB72, 'M', u'Ꭲ'), @@ -3970,6 +3955,10 @@ def _seg_38(): (0xAB78, 'M', u'Ꭸ'), (0xAB79, 'M', u'Ꭹ'), (0xAB7A, 'M', u'Ꭺ'), + ] + +def _seg_38(): + return [ (0xAB7B, 'M', u'Ꭻ'), (0xAB7C, 'M', u'Ꭼ'), (0xAB7D, 'M', u'Ꭽ'), @@ -4059,10 +4048,6 @@ def _seg_38(): (0xF907, 'M', u'龜'), (0xF909, 'M', u'契'), (0xF90A, 'M', u'金'), - ] - -def _seg_39(): - return [ (0xF90B, 'M', u'喇'), (0xF90C, 'M', u'奈'), (0xF90D, 'M', u'懶'), @@ -4074,6 +4059,10 @@ def _seg_39(): (0xF913, 'M', u'邏'), (0xF914, 'M', u'樂'), (0xF915, 'M', u'洛'), + ] + +def _seg_39(): + return [ (0xF916, 'M', u'烙'), (0xF917, 'M', u'珞'), (0xF918, 'M', u'落'), @@ -4163,10 +4152,6 @@ def _seg_39(): (0xF96C, 'M', u'塞'), (0xF96D, 'M', u'省'), (0xF96E, 'M', u'葉'), - ] - -def _seg_40(): - return [ (0xF96F, 'M', u'說'), (0xF970, 'M', u'殺'), (0xF971, 'M', u'辰'), @@ -4178,6 +4163,10 @@ def _seg_40(): (0xF977, 'M', u'亮'), (0xF978, 'M', u'兩'), (0xF979, 'M', u'凉'), + ] + +def _seg_40(): + return [ (0xF97A, 'M', u'梁'), (0xF97B, 'M', u'糧'), (0xF97C, 'M', u'良'), @@ -4267,10 +4256,6 @@ def _seg_40(): (0xF9D0, 'M', u'類'), (0xF9D1, 'M', u'六'), (0xF9D2, 'M', u'戮'), - ] - -def _seg_41(): - return [ (0xF9D3, 'M', u'陸'), (0xF9D4, 'M', u'倫'), (0xF9D5, 'M', u'崙'), @@ -4282,6 +4267,10 @@ def _seg_41(): (0xF9DB, 'M', u'率'), (0xF9DC, 'M', u'隆'), (0xF9DD, 'M', u'利'), + ] + +def _seg_41(): + return [ (0xF9DE, 'M', u'吏'), (0xF9DF, 'M', u'履'), (0xF9E0, 'M', u'易'), @@ -4371,10 +4360,6 @@ def _seg_41(): (0xFA39, 'M', u'塀'), (0xFA3A, 'M', u'墨'), (0xFA3B, 'M', u'層'), - ] - -def _seg_42(): - return [ (0xFA3C, 'M', u'屮'), (0xFA3D, 'M', u'悔'), (0xFA3E, 'M', u'慨'), @@ -4386,6 +4371,10 @@ def _seg_42(): (0xFA44, 'M', u'梅'), (0xFA45, 'M', u'海'), (0xFA46, 'M', u'渚'), + ] + +def _seg_42(): + return [ (0xFA47, 'M', u'漢'), (0xFA48, 'M', u'煮'), (0xFA49, 'M', u'爫'), @@ -4475,10 +4464,6 @@ def _seg_42(): (0xFA9F, 'M', u'犯'), (0xFAA0, 'M', u'猪'), (0xFAA1, 'M', u'瑱'), - ] - -def _seg_43(): - return [ (0xFAA2, 'M', u'甆'), (0xFAA3, 'M', u'画'), (0xFAA4, 'M', u'瘝'), @@ -4490,6 +4475,10 @@ def _seg_43(): (0xFAAA, 'M', u'着'), (0xFAAB, 'M', u'磌'), (0xFAAC, 'M', u'窱'), + ] + +def _seg_43(): + return [ (0xFAAD, 'M', u'節'), (0xFAAE, 'M', u'类'), (0xFAAF, 'M', u'絛'), @@ -4579,10 +4568,6 @@ def _seg_43(): (0xFB38, 'M', u'טּ'), (0xFB39, 'M', u'יּ'), (0xFB3A, 'M', u'ךּ'), - ] - -def _seg_44(): - return [ (0xFB3B, 'M', u'כּ'), (0xFB3C, 'M', u'לּ'), (0xFB3D, 'X'), @@ -4594,6 +4579,10 @@ def _seg_44(): (0xFB43, 'M', u'ףּ'), (0xFB44, 'M', u'פּ'), (0xFB45, 'X'), + ] + +def _seg_44(): + return [ (0xFB46, 'M', u'צּ'), (0xFB47, 'M', u'קּ'), (0xFB48, 'M', u'רּ'), @@ -4683,10 +4672,6 @@ def _seg_44(): (0xFC19, 'M', u'خج'), (0xFC1A, 'M', u'خح'), (0xFC1B, 'M', u'خم'), - ] - -def _seg_45(): - return [ (0xFC1C, 'M', u'سج'), (0xFC1D, 'M', u'سح'), (0xFC1E, 'M', u'سخ'), @@ -4698,6 +4683,10 @@ def _seg_45(): (0xFC24, 'M', u'ضخ'), (0xFC25, 'M', u'ضم'), (0xFC26, 'M', u'طح'), + ] + +def _seg_45(): + return [ (0xFC27, 'M', u'طم'), (0xFC28, 'M', u'ظم'), (0xFC29, 'M', u'عج'), @@ -4787,10 +4776,6 @@ def _seg_45(): (0xFC7D, 'M', u'في'), (0xFC7E, 'M', u'قى'), (0xFC7F, 'M', u'قي'), - ] - -def _seg_46(): - return [ (0xFC80, 'M', u'كا'), (0xFC81, 'M', u'كل'), (0xFC82, 'M', u'كم'), @@ -4802,6 +4787,10 @@ def _seg_46(): (0xFC88, 'M', u'ما'), (0xFC89, 'M', u'مم'), (0xFC8A, 'M', u'نر'), + ] + +def _seg_46(): + return [ (0xFC8B, 'M', u'نز'), (0xFC8C, 'M', u'نم'), (0xFC8D, 'M', u'نن'), @@ -4891,10 +4880,6 @@ def _seg_46(): (0xFCE1, 'M', u'بم'), (0xFCE2, 'M', u'به'), (0xFCE3, 'M', u'تم'), - ] - -def _seg_47(): - return [ (0xFCE4, 'M', u'ته'), (0xFCE5, 'M', u'ثم'), (0xFCE6, 'M', u'ثه'), @@ -4906,6 +4891,10 @@ def _seg_47(): (0xFCEC, 'M', u'كم'), (0xFCED, 'M', u'لم'), (0xFCEE, 'M', u'نم'), + ] + +def _seg_47(): + return [ (0xFCEF, 'M', u'نه'), (0xFCF0, 'M', u'يم'), (0xFCF1, 'M', u'يه'), @@ -4995,10 +4984,6 @@ def _seg_47(): (0xFD57, 'M', u'تمخ'), (0xFD58, 'M', u'جمح'), (0xFD5A, 'M', u'حمي'), - ] - -def _seg_48(): - return [ (0xFD5B, 'M', u'حمى'), (0xFD5C, 'M', u'سحج'), (0xFD5D, 'M', u'سجح'), @@ -5010,6 +4995,10 @@ def _seg_48(): (0xFD66, 'M', u'صمم'), (0xFD67, 'M', u'شحم'), (0xFD69, 'M', u'شجي'), + ] + +def _seg_48(): + return [ (0xFD6A, 'M', u'شمخ'), (0xFD6C, 'M', u'شمم'), (0xFD6E, 'M', u'ضحى'), @@ -5099,10 +5088,6 @@ def _seg_48(): (0xFDF3, 'M', u'اكبر'), (0xFDF4, 'M', u'محمد'), (0xFDF5, 'M', u'صلعم'), - ] - -def _seg_49(): - return [ (0xFDF6, 'M', u'رسول'), (0xFDF7, 'M', u'عليه'), (0xFDF8, 'M', u'وسلم'), @@ -5114,6 +5099,10 @@ def _seg_49(): (0xFDFE, 'X'), (0xFE00, 'I'), (0xFE10, '3', u','), + ] + +def _seg_49(): + return [ (0xFE11, 'M', u'、'), (0xFE12, 'X'), (0xFE13, '3', u':'), @@ -5203,10 +5192,6 @@ def _seg_49(): (0xFE8F, 'M', u'ب'), (0xFE93, 'M', u'ة'), (0xFE95, 'M', u'ت'), - ] - -def _seg_50(): - return [ (0xFE99, 'M', u'ث'), (0xFE9D, 'M', u'ج'), (0xFEA1, 'M', u'ح'), @@ -5218,6 +5203,10 @@ def _seg_50(): (0xFEB1, 'M', u'س'), (0xFEB5, 'M', u'ش'), (0xFEB9, 'M', u'ص'), + ] + +def _seg_50(): + return [ (0xFEBD, 'M', u'ض'), (0xFEC1, 'M', u'ط'), (0xFEC5, 'M', u'ظ'), @@ -5307,10 +5296,6 @@ def _seg_50(): (0xFF41, 'M', u'a'), (0xFF42, 'M', u'b'), (0xFF43, 'M', u'c'), - ] - -def _seg_51(): - return [ (0xFF44, 'M', u'd'), (0xFF45, 'M', u'e'), (0xFF46, 'M', u'f'), @@ -5322,6 +5307,10 @@ def _seg_51(): (0xFF4C, 'M', u'l'), (0xFF4D, 'M', u'm'), (0xFF4E, 'M', u'n'), + ] + +def _seg_51(): + return [ (0xFF4F, 'M', u'o'), (0xFF50, 'M', u'p'), (0xFF51, 'M', u'q'), @@ -5411,10 +5400,6 @@ def _seg_51(): (0xFFA5, 'M', u'ᆬ'), (0xFFA6, 'M', u'ᆭ'), (0xFFA7, 'M', u'ᄃ'), - ] - -def _seg_52(): - return [ (0xFFA8, 'M', u'ᄄ'), (0xFFA9, 'M', u'ᄅ'), (0xFFAA, 'M', u'ᆰ'), @@ -5426,6 +5411,10 @@ def _seg_52(): (0xFFB0, 'M', u'ᄚ'), (0xFFB1, 'M', u'ᄆ'), (0xFFB2, 'M', u'ᄇ'), + ] + +def _seg_52(): + return [ (0xFFB3, 'M', u'ᄈ'), (0xFFB4, 'M', u'ᄡ'), (0xFFB5, 'M', u'ᄉ'), @@ -5515,10 +5504,6 @@ def _seg_52(): (0x10300, 'V'), (0x10324, 'X'), (0x1032D, 'V'), - ] - -def _seg_53(): - return [ (0x1034B, 'X'), (0x10350, 'V'), (0x1037B, 'X'), @@ -5530,6 +5515,10 @@ def _seg_53(): (0x103D6, 'X'), (0x10400, 'M', u'𐐨'), (0x10401, 'M', u'𐐩'), + ] + +def _seg_53(): + return [ (0x10402, 'M', u'𐐪'), (0x10403, 'M', u'𐐫'), (0x10404, 'M', u'𐐬'), @@ -5619,10 +5608,6 @@ def _seg_53(): (0x10570, 'X'), (0x10600, 'V'), (0x10737, 'X'), - ] - -def _seg_54(): - return [ (0x10740, 'V'), (0x10756, 'X'), (0x10760, 'V'), @@ -5634,6 +5619,10 @@ def _seg_54(): (0x1080A, 'V'), (0x10836, 'X'), (0x10837, 'V'), + ] + +def _seg_54(): + return [ (0x10839, 'X'), (0x1083C, 'V'), (0x1083D, 'X'), @@ -5666,11 +5655,11 @@ def _seg_54(): (0x10A15, 'V'), (0x10A18, 'X'), (0x10A19, 'V'), - (0x10A34, 'X'), + (0x10A36, 'X'), (0x10A38, 'V'), (0x10A3B, 'X'), (0x10A3F, 'V'), - (0x10A48, 'X'), + (0x10A49, 'X'), (0x10A50, 'V'), (0x10A59, 'X'), (0x10A60, 'V'), @@ -5723,10 +5712,6 @@ def _seg_54(): (0x10C9B, 'M', u'𐳛'), (0x10C9C, 'M', u'𐳜'), (0x10C9D, 'M', u'𐳝'), - ] - -def _seg_55(): - return [ (0x10C9E, 'M', u'𐳞'), (0x10C9F, 'M', u'𐳟'), (0x10CA0, 'M', u'𐳠'), @@ -5738,6 +5723,10 @@ def _seg_55(): (0x10CA6, 'M', u'𐳦'), (0x10CA7, 'M', u'𐳧'), (0x10CA8, 'M', u'𐳨'), + ] + +def _seg_55(): + return [ (0x10CA9, 'M', u'𐳩'), (0x10CAA, 'M', u'𐳪'), (0x10CAB, 'M', u'𐳫'), @@ -5752,9 +5741,15 @@ def _seg_55(): (0x10CC0, 'V'), (0x10CF3, 'X'), (0x10CFA, 'V'), - (0x10D00, 'X'), + (0x10D28, 'X'), + (0x10D30, 'V'), + (0x10D3A, 'X'), (0x10E60, 'V'), (0x10E7F, 'X'), + (0x10F00, 'V'), + (0x10F28, 'X'), + (0x10F30, 'V'), + (0x10F5A, 'X'), (0x11000, 'V'), (0x1104E, 'X'), (0x11052, 'V'), @@ -5770,7 +5765,7 @@ def _seg_55(): (0x11100, 'V'), (0x11135, 'X'), (0x11136, 'V'), - (0x11144, 'X'), + (0x11147, 'X'), (0x11150, 'V'), (0x11177, 'X'), (0x11180, 'V'), @@ -5811,7 +5806,7 @@ def _seg_55(): (0x11334, 'X'), (0x11335, 'V'), (0x1133A, 'X'), - (0x1133C, 'V'), + (0x1133B, 'V'), (0x11345, 'X'), (0x11347, 'V'), (0x11349, 'X'), @@ -5827,16 +5822,16 @@ def _seg_55(): (0x1136D, 'X'), (0x11370, 'V'), (0x11375, 'X'), - ] - -def _seg_56(): - return [ (0x11400, 'V'), (0x1145A, 'X'), (0x1145B, 'V'), (0x1145C, 'X'), (0x1145D, 'V'), - (0x1145E, 'X'), + ] + +def _seg_56(): + return [ + (0x1145F, 'X'), (0x11480, 'V'), (0x114C8, 'X'), (0x114D0, 'V'), @@ -5856,11 +5851,13 @@ def _seg_56(): (0x116C0, 'V'), (0x116CA, 'X'), (0x11700, 'V'), - (0x1171A, 'X'), + (0x1171B, 'X'), (0x1171D, 'V'), (0x1172C, 'X'), (0x11730, 'V'), (0x11740, 'X'), + (0x11800, 'V'), + (0x1183C, 'X'), (0x118A0, 'M', u'𑣀'), (0x118A1, 'M', u'𑣁'), (0x118A2, 'M', u'𑣂'), @@ -5902,8 +5899,6 @@ def _seg_56(): (0x11A50, 'V'), (0x11A84, 'X'), (0x11A86, 'V'), - (0x11A9D, 'X'), - (0x11A9E, 'V'), (0x11AA3, 'X'), (0x11AC0, 'V'), (0x11AF9, 'X'), @@ -5931,14 +5926,28 @@ def _seg_56(): (0x11D3B, 'X'), (0x11D3C, 'V'), (0x11D3E, 'X'), - ] - -def _seg_57(): - return [ (0x11D3F, 'V'), (0x11D48, 'X'), (0x11D50, 'V'), (0x11D5A, 'X'), + (0x11D60, 'V'), + ] + +def _seg_57(): + return [ + (0x11D66, 'X'), + (0x11D67, 'V'), + (0x11D69, 'X'), + (0x11D6A, 'V'), + (0x11D8F, 'X'), + (0x11D90, 'V'), + (0x11D92, 'X'), + (0x11D93, 'V'), + (0x11D99, 'X'), + (0x11DA0, 'V'), + (0x11DAA, 'X'), + (0x11EE0, 'V'), + (0x11EF9, 'X'), (0x12000, 'V'), (0x1239A, 'X'), (0x12400, 'V'), @@ -5973,6 +5982,8 @@ def _seg_57(): (0x16B78, 'X'), (0x16B7D, 'V'), (0x16B90, 'X'), + (0x16E60, 'V'), + (0x16E9B, 'X'), (0x16F00, 'V'), (0x16F45, 'X'), (0x16F50, 'V'), @@ -5982,7 +5993,7 @@ def _seg_57(): (0x16FE0, 'V'), (0x16FE2, 'X'), (0x17000, 'V'), - (0x187ED, 'X'), + (0x187F2, 'X'), (0x18800, 'V'), (0x18AF3, 'X'), (0x1B000, 'V'), @@ -6024,21 +6035,23 @@ def _seg_57(): (0x1D1C1, 'V'), (0x1D1E9, 'X'), (0x1D200, 'V'), + ] + +def _seg_58(): + return [ (0x1D246, 'X'), + (0x1D2E0, 'V'), + (0x1D2F4, 'X'), (0x1D300, 'V'), (0x1D357, 'X'), (0x1D360, 'V'), - (0x1D372, 'X'), + (0x1D379, 'X'), (0x1D400, 'M', u'a'), (0x1D401, 'M', u'b'), (0x1D402, 'M', u'c'), (0x1D403, 'M', u'd'), (0x1D404, 'M', u'e'), (0x1D405, 'M', u'f'), - ] - -def _seg_58(): - return [ (0x1D406, 'M', u'g'), (0x1D407, 'M', u'h'), (0x1D408, 'M', u'i'), @@ -6126,6 +6139,10 @@ def _seg_58(): (0x1D45A, 'M', u'm'), (0x1D45B, 'M', u'n'), (0x1D45C, 'M', u'o'), + ] + +def _seg_59(): + return [ (0x1D45D, 'M', u'p'), (0x1D45E, 'M', u'q'), (0x1D45F, 'M', u'r'), @@ -6139,10 +6156,6 @@ def _seg_58(): (0x1D467, 'M', u'z'), (0x1D468, 'M', u'a'), (0x1D469, 'M', u'b'), - ] - -def _seg_59(): - return [ (0x1D46A, 'M', u'c'), (0x1D46B, 'M', u'd'), (0x1D46C, 'M', u'e'), @@ -6230,6 +6243,10 @@ def _seg_59(): (0x1D4C1, 'M', u'l'), (0x1D4C2, 'M', u'm'), (0x1D4C3, 'M', u'n'), + ] + +def _seg_60(): + return [ (0x1D4C4, 'X'), (0x1D4C5, 'M', u'p'), (0x1D4C6, 'M', u'q'), @@ -6243,10 +6260,6 @@ def _seg_59(): (0x1D4CE, 'M', u'y'), (0x1D4CF, 'M', u'z'), (0x1D4D0, 'M', u'a'), - ] - -def _seg_60(): - return [ (0x1D4D1, 'M', u'b'), (0x1D4D2, 'M', u'c'), (0x1D4D3, 'M', u'd'), @@ -6334,6 +6347,10 @@ def _seg_60(): (0x1D526, 'M', u'i'), (0x1D527, 'M', u'j'), (0x1D528, 'M', u'k'), + ] + +def _seg_61(): + return [ (0x1D529, 'M', u'l'), (0x1D52A, 'M', u'm'), (0x1D52B, 'M', u'n'), @@ -6347,10 +6364,6 @@ def _seg_60(): (0x1D533, 'M', u'v'), (0x1D534, 'M', u'w'), (0x1D535, 'M', u'x'), - ] - -def _seg_61(): - return [ (0x1D536, 'M', u'y'), (0x1D537, 'M', u'z'), (0x1D538, 'M', u'a'), @@ -6438,6 +6451,10 @@ def _seg_61(): (0x1D58C, 'M', u'g'), (0x1D58D, 'M', u'h'), (0x1D58E, 'M', u'i'), + ] + +def _seg_62(): + return [ (0x1D58F, 'M', u'j'), (0x1D590, 'M', u'k'), (0x1D591, 'M', u'l'), @@ -6451,10 +6468,6 @@ def _seg_61(): (0x1D599, 'M', u't'), (0x1D59A, 'M', u'u'), (0x1D59B, 'M', u'v'), - ] - -def _seg_62(): - return [ (0x1D59C, 'M', u'w'), (0x1D59D, 'M', u'x'), (0x1D59E, 'M', u'y'), @@ -6542,6 +6555,10 @@ def _seg_62(): (0x1D5F0, 'M', u'c'), (0x1D5F1, 'M', u'd'), (0x1D5F2, 'M', u'e'), + ] + +def _seg_63(): + return [ (0x1D5F3, 'M', u'f'), (0x1D5F4, 'M', u'g'), (0x1D5F5, 'M', u'h'), @@ -6555,10 +6572,6 @@ def _seg_62(): (0x1D5FD, 'M', u'p'), (0x1D5FE, 'M', u'q'), (0x1D5FF, 'M', u'r'), - ] - -def _seg_63(): - return [ (0x1D600, 'M', u's'), (0x1D601, 'M', u't'), (0x1D602, 'M', u'u'), @@ -6646,6 +6659,10 @@ def _seg_63(): (0x1D654, 'M', u'y'), (0x1D655, 'M', u'z'), (0x1D656, 'M', u'a'), + ] + +def _seg_64(): + return [ (0x1D657, 'M', u'b'), (0x1D658, 'M', u'c'), (0x1D659, 'M', u'd'), @@ -6659,10 +6676,6 @@ def _seg_63(): (0x1D661, 'M', u'l'), (0x1D662, 'M', u'm'), (0x1D663, 'M', u'n'), - ] - -def _seg_64(): - return [ (0x1D664, 'M', u'o'), (0x1D665, 'M', u'p'), (0x1D666, 'M', u'q'), @@ -6750,6 +6763,10 @@ def _seg_64(): (0x1D6B9, 'M', u'θ'), (0x1D6BA, 'M', u'σ'), (0x1D6BB, 'M', u'τ'), + ] + +def _seg_65(): + return [ (0x1D6BC, 'M', u'υ'), (0x1D6BD, 'M', u'φ'), (0x1D6BE, 'M', u'χ'), @@ -6763,10 +6780,6 @@ def _seg_64(): (0x1D6C6, 'M', u'ε'), (0x1D6C7, 'M', u'ζ'), (0x1D6C8, 'M', u'η'), - ] - -def _seg_65(): - return [ (0x1D6C9, 'M', u'θ'), (0x1D6CA, 'M', u'ι'), (0x1D6CB, 'M', u'κ'), @@ -6854,6 +6867,10 @@ def _seg_65(): (0x1D71F, 'M', u'δ'), (0x1D720, 'M', u'ε'), (0x1D721, 'M', u'ζ'), + ] + +def _seg_66(): + return [ (0x1D722, 'M', u'η'), (0x1D723, 'M', u'θ'), (0x1D724, 'M', u'ι'), @@ -6867,10 +6884,6 @@ def _seg_65(): (0x1D72C, 'M', u'ρ'), (0x1D72D, 'M', u'θ'), (0x1D72E, 'M', u'σ'), - ] - -def _seg_66(): - return [ (0x1D72F, 'M', u'τ'), (0x1D730, 'M', u'υ'), (0x1D731, 'M', u'φ'), @@ -6958,6 +6971,10 @@ def _seg_66(): (0x1D785, 'M', u'φ'), (0x1D786, 'M', u'χ'), (0x1D787, 'M', u'ψ'), + ] + +def _seg_67(): + return [ (0x1D788, 'M', u'ω'), (0x1D789, 'M', u'∂'), (0x1D78A, 'M', u'ε'), @@ -6971,10 +6988,6 @@ def _seg_66(): (0x1D792, 'M', u'γ'), (0x1D793, 'M', u'δ'), (0x1D794, 'M', u'ε'), - ] - -def _seg_67(): - return [ (0x1D795, 'M', u'ζ'), (0x1D796, 'M', u'η'), (0x1D797, 'M', u'θ'), @@ -7062,6 +7075,10 @@ def _seg_67(): (0x1D7EC, 'M', u'0'), (0x1D7ED, 'M', u'1'), (0x1D7EE, 'M', u'2'), + ] + +def _seg_68(): + return [ (0x1D7EF, 'M', u'3'), (0x1D7F0, 'M', u'4'), (0x1D7F1, 'M', u'5'), @@ -7075,10 +7092,6 @@ def _seg_67(): (0x1D7F9, 'M', u'3'), (0x1D7FA, 'M', u'4'), (0x1D7FB, 'M', u'5'), - ] - -def _seg_68(): - return [ (0x1D7FC, 'M', u'6'), (0x1D7FD, 'M', u'7'), (0x1D7FE, 'M', u'8'), @@ -7143,6 +7156,8 @@ def _seg_68(): (0x1E95A, 'X'), (0x1E95E, 'V'), (0x1E960, 'X'), + (0x1EC71, 'V'), + (0x1ECB5, 'X'), (0x1EE00, 'M', u'ا'), (0x1EE01, 'M', u'ب'), (0x1EE02, 'M', u'ج'), @@ -7164,6 +7179,10 @@ def _seg_68(): (0x1EE12, 'M', u'ق'), (0x1EE13, 'M', u'ر'), (0x1EE14, 'M', u'ش'), + ] + +def _seg_69(): + return [ (0x1EE15, 'M', u'ت'), (0x1EE16, 'M', u'ث'), (0x1EE17, 'M', u'خ'), @@ -7179,10 +7198,6 @@ def _seg_68(): (0x1EE21, 'M', u'ب'), (0x1EE22, 'M', u'ج'), (0x1EE23, 'X'), - ] - -def _seg_69(): - return [ (0x1EE24, 'M', u'ه'), (0x1EE25, 'X'), (0x1EE27, 'M', u'ح'), @@ -7268,6 +7283,10 @@ def _seg_69(): (0x1EE81, 'M', u'ب'), (0x1EE82, 'M', u'ج'), (0x1EE83, 'M', u'د'), + ] + +def _seg_70(): + return [ (0x1EE84, 'M', u'ه'), (0x1EE85, 'M', u'و'), (0x1EE86, 'M', u'ز'), @@ -7283,10 +7302,6 @@ def _seg_69(): (0x1EE90, 'M', u'ف'), (0x1EE91, 'M', u'ص'), (0x1EE92, 'M', u'ق'), - ] - -def _seg_70(): - return [ (0x1EE93, 'M', u'ر'), (0x1EE94, 'M', u'ش'), (0x1EE95, 'M', u'ت'), @@ -7372,6 +7387,10 @@ def _seg_70(): (0x1F122, '3', u'(s)'), (0x1F123, '3', u'(t)'), (0x1F124, '3', u'(u)'), + ] + +def _seg_71(): + return [ (0x1F125, '3', u'(v)'), (0x1F126, '3', u'(w)'), (0x1F127, '3', u'(x)'), @@ -7382,15 +7401,11 @@ def _seg_70(): (0x1F12C, 'M', u'r'), (0x1F12D, 'M', u'cd'), (0x1F12E, 'M', u'wz'), - (0x1F12F, 'X'), + (0x1F12F, 'V'), (0x1F130, 'M', u'a'), (0x1F131, 'M', u'b'), (0x1F132, 'M', u'c'), (0x1F133, 'M', u'd'), - ] - -def _seg_71(): - return [ (0x1F134, 'M', u'e'), (0x1F135, 'M', u'f'), (0x1F136, 'M', u'g'), @@ -7476,6 +7491,10 @@ def _seg_71(): (0x1F239, 'M', u'割'), (0x1F23A, 'M', u'営'), (0x1F23B, 'M', u'配'), + ] + +def _seg_72(): + return [ (0x1F23C, 'X'), (0x1F240, 'M', u'〔本〕'), (0x1F241, 'M', u'〔三〕'), @@ -7491,21 +7510,17 @@ def _seg_71(): (0x1F251, 'M', u'可'), (0x1F252, 'X'), (0x1F260, 'V'), - ] - -def _seg_72(): - return [ (0x1F266, 'X'), (0x1F300, 'V'), (0x1F6D5, 'X'), (0x1F6E0, 'V'), (0x1F6ED, 'X'), (0x1F6F0, 'V'), - (0x1F6F9, 'X'), + (0x1F6FA, 'X'), (0x1F700, 'V'), (0x1F774, 'X'), (0x1F780, 'V'), - (0x1F7D5, 'X'), + (0x1F7D9, 'X'), (0x1F800, 'V'), (0x1F80C, 'X'), (0x1F810, 'V'), @@ -7521,15 +7536,21 @@ def _seg_72(): (0x1F910, 'V'), (0x1F93F, 'X'), (0x1F940, 'V'), - (0x1F94D, 'X'), - (0x1F950, 'V'), - (0x1F96C, 'X'), - (0x1F980, 'V'), - (0x1F998, 'X'), + (0x1F971, 'X'), + (0x1F973, 'V'), + (0x1F977, 'X'), + (0x1F97A, 'V'), + (0x1F97B, 'X'), + (0x1F97C, 'V'), + (0x1F9A3, 'X'), + (0x1F9B0, 'V'), + (0x1F9BA, 'X'), (0x1F9C0, 'V'), - (0x1F9C1, 'X'), + (0x1F9C3, 'X'), (0x1F9D0, 'V'), - (0x1F9E7, 'X'), + (0x1FA00, 'X'), + (0x1FA60, 'V'), + (0x1FA6E, 'X'), (0x20000, 'V'), (0x2A6D7, 'X'), (0x2A700, 'V'), @@ -7574,6 +7595,10 @@ def _seg_72(): (0x2F81F, 'M', u'㓟'), (0x2F820, 'M', u'刻'), (0x2F821, 'M', u'剆'), + ] + +def _seg_73(): + return [ (0x2F822, 'M', u'割'), (0x2F823, 'M', u'剷'), (0x2F824, 'M', u'㔕'), @@ -7595,10 +7620,6 @@ def _seg_72(): (0x2F836, 'M', u'及'), (0x2F837, 'M', u'叟'), (0x2F838, 'M', u'𠭣'), - ] - -def _seg_73(): - return [ (0x2F839, 'M', u'叫'), (0x2F83A, 'M', u'叱'), (0x2F83B, 'M', u'吆'), @@ -7678,6 +7699,10 @@ def _seg_73(): (0x2F887, 'M', u'幩'), (0x2F888, 'M', u'㡢'), (0x2F889, 'M', u'𢆃'), + ] + +def _seg_74(): + return [ (0x2F88A, 'M', u'㡼'), (0x2F88B, 'M', u'庰'), (0x2F88C, 'M', u'庳'), @@ -7699,10 +7724,6 @@ def _seg_73(): (0x2F89E, 'M', u'志'), (0x2F89F, 'M', u'忹'), (0x2F8A0, 'M', u'悁'), - ] - -def _seg_74(): - return [ (0x2F8A1, 'M', u'㤺'), (0x2F8A2, 'M', u'㤜'), (0x2F8A3, 'M', u'悔'), @@ -7782,6 +7803,10 @@ def _seg_74(): (0x2F8ED, 'M', u'櫛'), (0x2F8EE, 'M', u'㰘'), (0x2F8EF, 'M', u'次'), + ] + +def _seg_75(): + return [ (0x2F8F0, 'M', u'𣢧'), (0x2F8F1, 'M', u'歔'), (0x2F8F2, 'M', u'㱎'), @@ -7803,10 +7828,6 @@ def _seg_74(): (0x2F902, 'M', u'流'), (0x2F903, 'M', u'浩'), (0x2F904, 'M', u'浸'), - ] - -def _seg_75(): - return [ (0x2F905, 'M', u'涅'), (0x2F906, 'M', u'𣴞'), (0x2F907, 'M', u'洴'), @@ -7886,6 +7907,10 @@ def _seg_75(): (0x2F953, 'M', u'祖'), (0x2F954, 'M', u'𥚚'), (0x2F955, 'M', u'𥛅'), + ] + +def _seg_76(): + return [ (0x2F956, 'M', u'福'), (0x2F957, 'M', u'秫'), (0x2F958, 'M', u'䄯'), @@ -7907,10 +7932,6 @@ def _seg_75(): (0x2F969, 'M', u'糣'), (0x2F96A, 'M', u'紀'), (0x2F96B, 'M', u'𥾆'), - ] - -def _seg_76(): - return [ (0x2F96C, 'M', u'絣'), (0x2F96D, 'M', u'䌁'), (0x2F96E, 'M', u'緇'), @@ -7990,6 +8011,10 @@ def _seg_76(): (0x2F9B8, 'M', u'蚈'), (0x2F9B9, 'M', u'蜎'), (0x2F9BA, 'M', u'蛢'), + ] + +def _seg_77(): + return [ (0x2F9BB, 'M', u'蝹'), (0x2F9BC, 'M', u'蜨'), (0x2F9BD, 'M', u'蝫'), @@ -8011,10 +8036,6 @@ def _seg_76(): (0x2F9CD, 'M', u'䚾'), (0x2F9CE, 'M', u'䛇'), (0x2F9CF, 'M', u'誠'), - ] - -def _seg_77(): - return [ (0x2F9D0, 'M', u'諭'), (0x2F9D1, 'M', u'變'), (0x2F9D2, 'M', u'豕'), @@ -8094,6 +8115,10 @@ def _seg_77(): (0x2FA1D, 'M', u'𪘀'), (0x2FA1E, 'X'), (0xE0100, 'I'), + ] + +def _seg_78(): + return [ (0xE01F0, 'X'), ] @@ -8176,4 +8201,5 @@ uts46data = tuple( + _seg_75() + _seg_76() + _seg_77() + + _seg_78() ) diff --git a/pipenv/patched/notpip/_vendor/packaging/__about__.py b/pipenv/patched/notpip/_vendor/packaging/__about__.py index 21fc6ce3..7481c9e2 100644 --- a/pipenv/patched/notpip/_vendor/packaging/__about__.py +++ b/pipenv/patched/notpip/_vendor/packaging/__about__.py @@ -4,18 +4,24 @@ from __future__ import absolute_import, division, print_function __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "18.0" +__version__ = "19.0" __author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" __license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2014-2018 %s" % __author__ +__copyright__ = "Copyright 2014-2019 %s" % __author__ diff --git a/pipenv/patched/notpip/_vendor/packaging/__init__.py b/pipenv/patched/notpip/_vendor/packaging/__init__.py index 5ee62202..a0cf67df 100644 --- a/pipenv/patched/notpip/_vendor/packaging/__init__.py +++ b/pipenv/patched/notpip/_vendor/packaging/__init__.py @@ -4,11 +4,23 @@ from __future__ import absolute_import, division, print_function from .__about__ import ( - __author__, __copyright__, __email__, __license__, __summary__, __title__, - __uri__, __version__ + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + __version__, ) __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] diff --git a/pipenv/patched/notpip/_vendor/packaging/_compat.py b/pipenv/patched/notpip/_vendor/packaging/_compat.py index 210bb80b..25da473c 100644 --- a/pipenv/patched/notpip/_vendor/packaging/_compat.py +++ b/pipenv/patched/notpip/_vendor/packaging/_compat.py @@ -12,9 +12,9 @@ PY3 = sys.version_info[0] == 3 # flake8: noqa if PY3: - string_types = str, + string_types = (str,) else: - string_types = basestring, + string_types = (basestring,) def with_metaclass(meta, *bases): @@ -27,4 +27,5 @@ def with_metaclass(meta, *bases): class metaclass(meta): def __new__(cls, name, this_bases, d): return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + return type.__new__(metaclass, "temporary_class", (), {}) diff --git a/pipenv/patched/notpip/_vendor/packaging/_structures.py b/pipenv/patched/notpip/_vendor/packaging/_structures.py index e9fc4a04..68dcca63 100644 --- a/pipenv/patched/notpip/_vendor/packaging/_structures.py +++ b/pipenv/patched/notpip/_vendor/packaging/_structures.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function class Infinity(object): - def __repr__(self): return "Infinity" @@ -38,7 +37,6 @@ Infinity = Infinity() class NegativeInfinity(object): - def __repr__(self): return "-Infinity" diff --git a/pipenv/patched/notpip/_vendor/packaging/markers.py b/pipenv/patched/notpip/_vendor/packaging/markers.py index dc3cef81..50a08091 100644 --- a/pipenv/patched/notpip/_vendor/packaging/markers.py +++ b/pipenv/patched/notpip/_vendor/packaging/markers.py @@ -17,8 +17,11 @@ from .specifiers import Specifier, InvalidSpecifier __all__ = [ - "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName", - "Marker", "default_environment", + "InvalidMarker", + "UndefinedComparison", + "UndefinedEnvironmentName", + "Marker", + "default_environment", ] @@ -42,7 +45,6 @@ class UndefinedEnvironmentName(ValueError): class Node(object): - def __init__(self, value): self.value = value @@ -57,62 +59,52 @@ class Node(object): class Variable(Node): - def serialize(self): return str(self) class Value(Node): - def serialize(self): return '"{0}"'.format(self) class Op(Node): - def serialize(self): return str(self) VARIABLE = ( - L("implementation_version") | - L("platform_python_implementation") | - L("implementation_name") | - L("python_full_version") | - L("platform_release") | - L("platform_version") | - L("platform_machine") | - L("platform_system") | - L("python_version") | - L("sys_platform") | - L("os_name") | - L("os.name") | # PEP-345 - L("sys.platform") | # PEP-345 - L("platform.version") | # PEP-345 - L("platform.machine") | # PEP-345 - L("platform.python_implementation") | # PEP-345 - L("python_implementation") | # undocumented setuptools legacy - L("extra") + L("implementation_version") + | L("platform_python_implementation") + | L("implementation_name") + | L("python_full_version") + | L("platform_release") + | L("platform_version") + | L("platform_machine") + | L("platform_system") + | L("python_version") + | L("sys_platform") + | L("os_name") + | L("os.name") + | L("sys.platform") # PEP-345 + | L("platform.version") # PEP-345 + | L("platform.machine") # PEP-345 + | L("platform.python_implementation") # PEP-345 + | L("python_implementation") # PEP-345 + | L("extra") # undocumented setuptools legacy ) ALIASES = { - 'os.name': 'os_name', - 'sys.platform': 'sys_platform', - 'platform.version': 'platform_version', - 'platform.machine': 'platform_machine', - 'platform.python_implementation': 'platform_python_implementation', - 'python_implementation': 'platform_python_implementation' + "os.name": "os_name", + "sys.platform": "sys_platform", + "platform.version": "platform_version", + "platform.machine": "platform_machine", + "platform.python_implementation": "platform_python_implementation", + "python_implementation": "platform_python_implementation", } VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) VERSION_CMP = ( - L("===") | - L("==") | - L(">=") | - L("<=") | - L("!=") | - L("~=") | - L(">") | - L("<") + L("===") | L("==") | L(">=") | L("<=") | L("!=") | L("~=") | L(">") | L("<") ) MARKER_OP = VERSION_CMP | L("not in") | L("in") @@ -152,8 +144,11 @@ def _format_marker(marker, first=True): # where the single item is itself it's own list. In that case we want skip # the rest of this function so that we don't get extraneous () on the # outside. - if (isinstance(marker, list) and len(marker) == 1 and - isinstance(marker[0], (list, tuple))): + if ( + isinstance(marker, list) + and len(marker) == 1 + and isinstance(marker[0], (list, tuple)) + ): return _format_marker(marker[0]) if isinstance(marker, list): @@ -239,20 +234,20 @@ def _evaluate_markers(markers, environment): def format_full_version(info): - version = '{0.major}.{0.minor}.{0.micro}'.format(info) + version = "{0.major}.{0.minor}.{0.micro}".format(info) kind = info.releaselevel - if kind != 'final': + if kind != "final": version += kind[0] + str(info.serial) return version def default_environment(): - if hasattr(sys, 'implementation'): + if hasattr(sys, "implementation"): iver = format_full_version(sys.implementation.version) implementation_name = sys.implementation.name else: - iver = '0' - implementation_name = '' + iver = "0" + implementation_name = "" return { "implementation_name": implementation_name, @@ -270,13 +265,13 @@ def default_environment(): class Marker(object): - def __init__(self, marker): try: self._markers = _coerce_parse_result(MARKER.parseString(marker)) except ParseException as e: err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( - marker, marker[e.loc:e.loc + 8]) + marker, marker[e.loc : e.loc + 8] + ) raise InvalidMarker(err_str) def __str__(self): diff --git a/pipenv/patched/notpip/_vendor/packaging/requirements.py b/pipenv/patched/notpip/_vendor/packaging/requirements.py index 5ec5d74a..a3de7673 100644 --- a/pipenv/patched/notpip/_vendor/packaging/requirements.py +++ b/pipenv/patched/notpip/_vendor/packaging/requirements.py @@ -38,8 +38,8 @@ IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) NAME = IDENTIFIER("name") EXTRA = IDENTIFIER -URI = Regex(r'[^ ]+')("url") -URL = (AT + URI) +URI = Regex(r"[^ ]+")("url") +URL = AT + URI EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") @@ -48,17 +48,18 @@ VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY -VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), - joinString=",", adjacent=False)("_raw_spec") +VERSION_MANY = Combine( + VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), joinString=",", adjacent=False +)("_raw_spec") _VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)) -_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '') +_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or "") VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") MARKER_EXPR.setParseAction( - lambda s, l, t: Marker(s[t._original_start:t._original_end]) + lambda s, l, t: Marker(s[t._original_start : t._original_end]) ) MARKER_SEPARATOR = SEMICOLON MARKER = MARKER_SEPARATOR + MARKER_EXPR @@ -66,8 +67,7 @@ MARKER = MARKER_SEPARATOR + MARKER_EXPR VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) URL_AND_MARKER = URL + Optional(MARKER) -NAMED_REQUIREMENT = \ - NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) +NAMED_REQUIREMENT = NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd # pyparsing isn't thread safe during initialization, so we do it eagerly, see @@ -92,15 +92,21 @@ class Requirement(object): try: req = REQUIREMENT.parseString(requirement_string) except ParseException as e: - raise InvalidRequirement("Parse error at \"{0!r}\": {1}".format( - requirement_string[e.loc:e.loc + 8], e.msg - )) + raise InvalidRequirement( + 'Parse error at "{0!r}": {1}'.format( + requirement_string[e.loc : e.loc + 8], e.msg + ) + ) self.name = req.name if req.url: parsed_url = urlparse.urlparse(req.url) - if not (parsed_url.scheme and parsed_url.netloc) or ( - not parsed_url.scheme and not parsed_url.netloc): + if parsed_url.scheme == "file": + if urlparse.urlunparse(parsed_url) != req.url: + raise InvalidRequirement("Invalid URL given") + elif not (parsed_url.scheme and parsed_url.netloc) or ( + not parsed_url.scheme and not parsed_url.netloc + ): raise InvalidRequirement("Invalid URL: {0}".format(req.url)) self.url = req.url else: @@ -120,6 +126,8 @@ class Requirement(object): if self.url: parts.append("@ {0}".format(self.url)) + if self.marker: + parts.append(" ") if self.marker: parts.append("; {0}".format(self.marker)) diff --git a/pipenv/patched/notpip/_vendor/packaging/specifiers.py b/pipenv/patched/notpip/_vendor/packaging/specifiers.py index 4c798999..743576a0 100644 --- a/pipenv/patched/notpip/_vendor/packaging/specifiers.py +++ b/pipenv/patched/notpip/_vendor/packaging/specifiers.py @@ -19,7 +19,6 @@ class InvalidSpecifier(ValueError): class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): - @abc.abstractmethod def __str__(self): """ @@ -84,10 +83,7 @@ class _IndividualSpecifier(BaseSpecifier): if not match: raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec)) - self._spec = ( - match.group("operator").strip(), - match.group("version").strip(), - ) + self._spec = (match.group("operator").strip(), match.group("version").strip()) # Store whether or not this Specifier should accept prereleases self._prereleases = prereleases @@ -99,11 +95,7 @@ class _IndividualSpecifier(BaseSpecifier): else "" ) - return "<{0}({1!r}{2})>".format( - self.__class__.__name__, - str(self), - pre, - ) + return "<{0}({1!r}{2})>".format(self.__class__.__name__, str(self), pre) def __str__(self): return "{0}{1}".format(*self._spec) @@ -194,8 +186,9 @@ class _IndividualSpecifier(BaseSpecifier): # If our version is a prerelease, and we were not set to allow # prereleases, then we'll store it for later incase nothing # else matches this specifier. - if (parsed_version.is_prerelease and not - (prereleases or self.prereleases)): + if parsed_version.is_prerelease and not ( + prereleases or self.prereleases + ): found_prereleases.append(version) # Either this is not a prerelease, or we should have been # accepting prereleases from the beginning. @@ -213,8 +206,7 @@ class _IndividualSpecifier(BaseSpecifier): class LegacySpecifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P(==|!=|<=|>=|<|>)) \s* (?P @@ -225,10 +217,8 @@ class LegacySpecifier(_IndividualSpecifier): # them, and a comma since it's a version separator. ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "==": "equal", @@ -269,13 +259,13 @@ def _require_version_compare(fn): if not isinstance(prospective, Version): return False return fn(self, prospective, spec) + return wrapped class Specifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P(~=|==|!=|<=|>=|<|>|===)) (?P (?: @@ -367,10 +357,8 @@ class Specifier(_IndividualSpecifier): ) ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "~=": "compatible", @@ -397,8 +385,7 @@ class Specifier(_IndividualSpecifier): prefix = ".".join( list( itertools.takewhile( - lambda x: (not x.startswith("post") and not - x.startswith("dev")), + lambda x: (not x.startswith("post") and not x.startswith("dev")), _version_split(spec), ) )[:-1] @@ -407,8 +394,9 @@ class Specifier(_IndividualSpecifier): # Add the prefix notation to the end of our string prefix += ".*" - return (self._get_operator(">=")(prospective, spec) and - self._get_operator("==")(prospective, prefix)) + return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( + prospective, prefix + ) @_require_version_compare def _compare_equal(self, prospective, spec): @@ -428,7 +416,7 @@ class Specifier(_IndividualSpecifier): # Shorten the prospective version to be the same length as the spec # so that we can determine if the specifier is a prefix of the # prospective version or not. - prospective = prospective[:len(spec)] + prospective = prospective[: len(spec)] # Pad out our two sides with zeros so that they both equal the same # length. @@ -567,27 +555,17 @@ def _pad_version(left, right): right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) # Get the rest of our versions - left_split.append(left[len(left_split[0]):]) - right_split.append(right[len(right_split[0]):]) + left_split.append(left[len(left_split[0]) :]) + right_split.append(right[len(right_split[0]) :]) # Insert our padding - left_split.insert( - 1, - ["0"] * max(0, len(right_split[0]) - len(left_split[0])), - ) - right_split.insert( - 1, - ["0"] * max(0, len(left_split[0]) - len(right_split[0])), - ) + left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) + right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) - return ( - list(itertools.chain(*left_split)), - list(itertools.chain(*right_split)), - ) + return (list(itertools.chain(*left_split)), list(itertools.chain(*right_split))) class SpecifierSet(BaseSpecifier): - def __init__(self, specifiers="", prereleases=None): # Split on , to break each indidivual specifier into it's own item, and # strip each item to remove leading/trailing whitespace. @@ -721,10 +699,7 @@ class SpecifierSet(BaseSpecifier): # given version is contained within all of them. # Note: This use of all() here means that an empty set of specifiers # will always return True, this is an explicit design decision. - return all( - s.contains(item, prereleases=prereleases) - for s in self._specs - ) + return all(s.contains(item, prereleases=prereleases) for s in self._specs) def filter(self, iterable, prereleases=None): # Determine if we're forcing a prerelease or not, if we're not forcing diff --git a/pipenv/patched/notpip/_vendor/packaging/utils.py b/pipenv/patched/notpip/_vendor/packaging/utils.py index 4b94a82f..88418786 100644 --- a/pipenv/patched/notpip/_vendor/packaging/utils.py +++ b/pipenv/patched/notpip/_vendor/packaging/utils.py @@ -36,13 +36,7 @@ def canonicalize_version(version): # Release segment # NB: This strips trailing '.0's to normalize - parts.append( - re.sub( - r'(\.0)+$', - '', - ".".join(str(x) for x in version.release) - ) - ) + parts.append(re.sub(r"(\.0)+$", "", ".".join(str(x) for x in version.release))) # Pre-release if version.pre is not None: diff --git a/pipenv/patched/notpip/_vendor/packaging/version.py b/pipenv/patched/notpip/_vendor/packaging/version.py index 6ed5cbbd..95157a1f 100644 --- a/pipenv/patched/notpip/_vendor/packaging/version.py +++ b/pipenv/patched/notpip/_vendor/packaging/version.py @@ -10,14 +10,11 @@ import re from ._structures import Infinity -__all__ = [ - "parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN" -] +__all__ = ["parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN"] _Version = collections.namedtuple( - "_Version", - ["epoch", "release", "dev", "pre", "post", "local"], + "_Version", ["epoch", "release", "dev", "pre", "post", "local"] ) @@ -40,7 +37,6 @@ class InvalidVersion(ValueError): class _BaseVersion(object): - def __hash__(self): return hash(self._key) @@ -70,7 +66,6 @@ class _BaseVersion(object): class LegacyVersion(_BaseVersion): - def __init__(self, version): self._version = str(version) self._key = _legacy_cmpkey(self._version) @@ -126,12 +121,14 @@ class LegacyVersion(_BaseVersion): return False -_legacy_version_component_re = re.compile( - r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE, -) +_legacy_version_component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE) _legacy_version_replacement_map = { - "pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@", + "pre": "c", + "preview": "c", + "-": "final-", + "rc": "c", + "dev": "@", } @@ -215,10 +212,7 @@ VERSION_PATTERN = r""" class Version(_BaseVersion): - _regex = re.compile( - r"^\s*" + VERSION_PATTERN + r"\s*$", - re.VERBOSE | re.IGNORECASE, - ) + _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE) def __init__(self, version): # Validate the version and parse it into pieces @@ -230,18 +224,11 @@ class Version(_BaseVersion): self._version = _Version( epoch=int(match.group("epoch")) if match.group("epoch") else 0, release=tuple(int(i) for i in match.group("release").split(".")), - pre=_parse_letter_version( - match.group("pre_l"), - match.group("pre_n"), - ), + pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")), post=_parse_letter_version( - match.group("post_l"), - match.group("post_n1") or match.group("post_n2"), - ), - dev=_parse_letter_version( - match.group("dev_l"), - match.group("dev_n"), + match.group("post_l"), match.group("post_n1") or match.group("post_n2") ), + dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")), local=_parse_local_version(match.group("local")), ) @@ -395,12 +382,7 @@ def _cmpkey(epoch, release, pre, post, dev, local): # re-reverse it back into the correct order and make it a tuple and use # that for our sorting key. release = tuple( - reversed(list( - itertools.dropwhile( - lambda x: x == 0, - reversed(release), - ) - )) + reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release)))) ) # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. @@ -433,9 +415,6 @@ def _cmpkey(epoch, release, pre, post, dev, local): # - Numeric segments sort numerically # - Shorter versions sort before longer versions when the prefixes # match exactly - local = tuple( - (i, "") if isinstance(i, int) else (-Infinity, i) - for i in local - ) + local = tuple((i, "") if isinstance(i, int) else (-Infinity, i) for i in local) return epoch, release, pre, post, dev, local diff --git a/pipenv/patched/notpip/_vendor/pep517/__init__.py b/pipenv/patched/notpip/_vendor/pep517/__init__.py index 8beedea4..9c1a098f 100644 --- a/pipenv/patched/notpip/_vendor/pep517/__init__.py +++ b/pipenv/patched/notpip/_vendor/pep517/__init__.py @@ -1,4 +1,4 @@ """Wrappers to build Python packages using PEP 517 hooks """ -__version__ = '0.2' +__version__ = '0.5.0' diff --git a/pipenv/patched/notpip/_vendor/pep517/_in_process.py b/pipenv/patched/notpip/_vendor/pep517/_in_process.py index baa14d38..d6524b66 100644 --- a/pipenv/patched/notpip/_vendor/pep517/_in_process.py +++ b/pipenv/patched/notpip/_vendor/pep517/_in_process.py @@ -21,19 +21,28 @@ import sys # This is run as a script, not a module, so it can't do a relative import import compat + +class BackendUnavailable(Exception): + """Raised if we cannot import the backend""" + + def _build_backend(): """Find and load the build backend""" ep = os.environ['PEP517_BUILD_BACKEND'] mod_path, _, obj_path = ep.partition(':') - obj = import_module(mod_path) + try: + obj = import_module(mod_path) + except ImportError: + raise BackendUnavailable if obj_path: for path_part in obj_path.split('.'): obj = getattr(obj, path_part) return obj + def get_requires_for_build_wheel(config_settings): """Invoke the optional get_requires_for_build_wheel hook - + Returns [] if the hook is not defined. """ backend = _build_backend() @@ -44,9 +53,10 @@ def get_requires_for_build_wheel(config_settings): else: return hook(config_settings) + def prepare_metadata_for_build_wheel(metadata_directory, config_settings): """Invoke optional prepare_metadata_for_build_wheel - + Implements a fallback by building a wheel if the hook isn't defined. """ backend = _build_backend() @@ -58,8 +68,10 @@ def prepare_metadata_for_build_wheel(metadata_directory, config_settings): else: return hook(metadata_directory, config_settings) + WHEEL_BUILT_MARKER = 'PEP517_ALREADY_BUILT_WHEEL' + def _dist_info_files(whl_zip): """Identify the .dist-info folder inside a wheel ZipFile.""" res = [] @@ -71,11 +83,13 @@ def _dist_info_files(whl_zip): return res raise Exception("No .dist-info folder found in wheel") -def _get_wheel_metadata_from_wheel(backend, metadata_directory, config_settings): + +def _get_wheel_metadata_from_wheel( + backend, metadata_directory, config_settings): """Build a wheel and extract the metadata from it. - - Fallback for when the build backend does not define the 'get_wheel_metadata' - hook. + + Fallback for when the build backend does not + define the 'get_wheel_metadata' hook. """ from zipfile import ZipFile whl_basename = backend.build_wheel(metadata_directory, config_settings) @@ -88,6 +102,7 @@ def _get_wheel_metadata_from_wheel(backend, metadata_directory, config_settings) zipf.extractall(path=metadata_directory, members=dist_info) return dist_info[0].split('/')[0] + def _find_already_built_wheel(metadata_directory): """Check for a wheel already built during the get_wheel_metadata hook. """ @@ -105,14 +120,16 @@ def _find_already_built_wheel(metadata_directory): print('Found multiple .whl files; unspecified behaviour. ' 'Will call build_wheel.') return None - + # Exactly one .whl file return whl_files[0] + def build_wheel(wheel_directory, config_settings, metadata_directory=None): """Invoke the mandatory build_wheel hook. - - If a wheel was already built in the prepare_metadata_for_build_wheel fallback, this + + If a wheel was already built in the + prepare_metadata_for_build_wheel fallback, this will copy it rather than rebuilding the wheel. """ prebuilt_whl = _find_already_built_wheel(metadata_directory) @@ -137,12 +154,15 @@ def get_requires_for_build_sdist(config_settings): else: return hook(config_settings) + class _DummyException(Exception): """Nothing should ever raise this exception""" + class GotUnsupportedOperation(Exception): """For internal use when backend raises UnsupportedOperation""" + def build_sdist(sdist_directory, config_settings): """Invoke the mandatory build_sdist hook.""" backend = _build_backend() @@ -151,6 +171,7 @@ def build_sdist(sdist_directory, config_settings): except getattr(backend, 'UnsupportedOperation', _DummyException): raise GotUnsupportedOperation + HOOK_NAMES = { 'get_requires_for_build_wheel', 'prepare_metadata_for_build_wheel', @@ -159,6 +180,7 @@ HOOK_NAMES = { 'build_sdist', } + def main(): if len(sys.argv) < 3: sys.exit("Needs args: hook_name, control_dir") @@ -173,10 +195,13 @@ def main(): json_out = {'unsupported': False, 'return_val': None} try: json_out['return_val'] = hook(**hook_input['kwargs']) + except BackendUnavailable: + json_out['no_backend'] = True except GotUnsupportedOperation: json_out['unsupported'] = True - + compat.write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) + if __name__ == '__main__': main() diff --git a/pipenv/patched/notpip/_vendor/pep517/build.py b/pipenv/patched/notpip/_vendor/pep517/build.py new file mode 100644 index 00000000..db9a0799 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/build.py @@ -0,0 +1,108 @@ +"""Build a project using PEP 517 hooks. +""" +import argparse +import logging +import os +import contextlib +from pipenv.patched.notpip._vendor import pytoml +import shutil +import errno +import tempfile + +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + + +@contextlib.contextmanager +def tempdir(): + td = tempfile.mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + + +def _do_build(hooks, env, dist, dest): + get_requires_name = 'get_requires_for_build_{dist}'.format(**locals()) + get_requires = getattr(hooks, get_requires_name) + reqs = get_requires({}) + log.info('Got build requires: %s', reqs) + + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + + with tempdir() as td: + log.info('Trying to build %s in %s', dist, td) + build_name = 'build_{dist}'.format(**locals()) + build = getattr(hooks, build_name) + filename = build(td, {}) + source = os.path.join(td, filename) + shutil.move(source, os.path.join(dest, os.path.basename(filename))) + + +def mkdir_p(*args, **kwargs): + """Like `mkdir`, but does not raise an exception if the + directory already exists. + """ + try: + return os.mkdir(*args, **kwargs) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + +def build(source_dir, dist, dest=None): + pyproject = os.path.join(source_dir, 'pyproject.toml') + dest = os.path.join(source_dir, dest or 'dist') + mkdir_p(dest) + + with open(pyproject) as f: + pyproject_data = pytoml.load(f) + # Ensure the mandatory data can be loaded + buildsys = pyproject_data['build-system'] + requires = buildsys['requires'] + backend = buildsys['build-backend'] + + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + _do_build(hooks, env, dist, dest) + + +parser = argparse.ArgumentParser() +parser.add_argument( + 'source_dir', + help="A directory containing pyproject.toml", +) +parser.add_argument( + '--binary', '-b', + action='store_true', + default=False, +) +parser.add_argument( + '--source', '-s', + action='store_true', + default=False, +) +parser.add_argument( + '--out-dir', '-o', + help="Destination in which to save the builds relative to source dir", +) + + +def main(args): + # determine which dists to build + dists = list(filter(None, ( + 'sdist' if args.source or not args.binary else None, + 'wheel' if args.binary or not args.source else None, + ))) + + for dist in dists: + build(args.source_dir, dist, args.out_dir) + + +if __name__ == '__main__': + main(parser.parse_args()) diff --git a/pipenv/patched/notpip/_vendor/pep517/check.py b/pipenv/patched/notpip/_vendor/pep517/check.py index 3dffd2e0..9d28ba44 100644 --- a/pipenv/patched/notpip/_vendor/pep517/check.py +++ b/pipenv/patched/notpip/_vendor/pep517/check.py @@ -18,10 +18,11 @@ from .wrappers import Pep517HookCaller log = logging.getLogger(__name__) -def check_build_sdist(hooks): + +def check_build_sdist(hooks, build_sys_requires): with BuildEnvironment() as env: try: - env.pip_install(hooks.build_sys_requires) + env.pip_install(build_sys_requires) log.info('Installed static build dependencies') except CalledProcessError: log.error('Failed to install static build dependencies') @@ -30,7 +31,7 @@ def check_build_sdist(hooks): try: reqs = hooks.get_requires_for_build_sdist({}) log.info('Got build requires: %s', reqs) - except: + except Exception: log.error('Failure in get_requires_for_build_sdist', exc_info=True) return False @@ -47,12 +48,13 @@ def check_build_sdist(hooks): try: filename = hooks.build_sdist(td, {}) log.info('build_sdist returned %r', filename) - except: + except Exception: log.info('Failure in build_sdist', exc_info=True) return False if not filename.endswith('.tar.gz'): - log.error("Filename %s doesn't have .tar.gz extension", filename) + log.error( + "Filename %s doesn't have .tar.gz extension", filename) return False path = pjoin(td, filename) @@ -73,10 +75,11 @@ def check_build_sdist(hooks): return True -def check_build_wheel(hooks): + +def check_build_wheel(hooks, build_sys_requires): with BuildEnvironment() as env: try: - env.pip_install(hooks.build_sys_requires) + env.pip_install(build_sys_requires) log.info('Installed static build dependencies') except CalledProcessError: log.error('Failed to install static build dependencies') @@ -85,7 +88,7 @@ def check_build_wheel(hooks): try: reqs = hooks.get_requires_for_build_wheel({}) log.info('Got build requires: %s', reqs) - except: + except Exception: log.error('Failure in get_requires_for_build_sdist', exc_info=True) return False @@ -102,7 +105,7 @@ def check_build_wheel(hooks): try: filename = hooks.build_wheel(td, {}) log.info('build_wheel returned %r', filename) - except: + except Exception: log.info('Failure in build_wheel', exc_info=True) return False @@ -151,8 +154,8 @@ def check(source_dir): hooks = Pep517HookCaller(source_dir, backend) - sdist_ok = check_build_sdist(hooks) - wheel_ok = check_build_wheel(hooks) + sdist_ok = check_build_sdist(hooks, requires) + wheel_ok = check_build_wheel(hooks, requires) if not sdist_ok: log.warning('Sdist checks failed; scroll up to see') @@ -164,7 +167,8 @@ def check(source_dir): def main(argv=None): ap = argparse.ArgumentParser() - ap.add_argument('source_dir', + ap.add_argument( + 'source_dir', help="A directory containing pyproject.toml") args = ap.parse_args(argv) @@ -178,17 +182,21 @@ def main(argv=None): print(ansi('Checks failed', 'red')) sys.exit(1) + ansi_codes = { 'reset': '\x1b[0m', 'bold': '\x1b[1m', 'red': '\x1b[31m', 'green': '\x1b[32m', } + + def ansi(s, attr): if os.name != 'nt' and sys.stdout.isatty(): return ansi_codes[attr] + str(s) + ansi_codes['reset'] else: return str(s) + if __name__ == '__main__': main() diff --git a/pipenv/patched/notpip/_vendor/pep517/colorlog.py b/pipenv/patched/notpip/_vendor/pep517/colorlog.py index 26cf7480..69c8a59d 100644 --- a/pipenv/patched/notpip/_vendor/pep517/colorlog.py +++ b/pipenv/patched/notpip/_vendor/pep517/colorlog.py @@ -24,6 +24,7 @@ try: except ImportError: curses = None + def _stderr_supports_color(): color = False if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): @@ -35,13 +36,14 @@ def _stderr_supports_color(): pass return color + class LogFormatter(logging.Formatter): """Log formatter with colour support """ DEFAULT_COLORS = { - logging.INFO: 2, # Green - logging.WARNING: 3, # Yellow - logging.ERROR: 1, # Red + logging.INFO: 2, # Green + logging.WARNING: 3, # Yellow + logging.ERROR: 1, # Red logging.CRITICAL: 1, } @@ -75,7 +77,8 @@ class LogFormatter(logging.Formatter): fg_color = str(fg_color, "ascii") for levelno, code in self.DEFAULT_COLORS.items(): - self._colors[levelno] = str(curses.tparm(fg_color, code), "ascii") + self._colors[levelno] = str( + curses.tparm(fg_color, code), "ascii") self._normal = str(curses.tigetstr("sgr0"), "ascii") scr = curses.initscr() @@ -83,15 +86,16 @@ class LogFormatter(logging.Formatter): curses.endwin() else: self._normal = '' - # Default width is usually 80, but too wide is worse than too narrow + # Default width is usually 80, but too wide is + # worse than too narrow self.termwidth = 70 def formatMessage(self, record): - l = len(record.message) + mlen = len(record.message) right_text = '{initial}-{name}'.format(initial=record.levelname[0], name=record.name) - if l + len(right_text) < self.termwidth: - space = ' ' * (self.termwidth - (l + len(right_text))) + if mlen + len(right_text) < self.termwidth: + space = ' ' * (self.termwidth - (mlen + len(right_text))) else: space = ' ' @@ -103,6 +107,7 @@ class LogFormatter(logging.Formatter): return record.message + space + start_color + right_text + end_color + def enable_colourful_output(level=logging.INFO): handler = logging.StreamHandler() handler.setFormatter(LogFormatter()) diff --git a/pipenv/patched/notpip/_vendor/pep517/envbuild.py b/pipenv/patched/notpip/_vendor/pep517/envbuild.py index c54d3585..8a5ad4d7 100644 --- a/pipenv/patched/notpip/_vendor/pep517/envbuild.py +++ b/pipenv/patched/notpip/_vendor/pep517/envbuild.py @@ -14,6 +14,7 @@ from .wrappers import Pep517HookCaller log = logging.getLogger(__name__) + def _load_pyproject(source_dir): with open(os.path.join(source_dir, 'pyproject.toml')) as f: pyproject_data = pytoml.load(f) @@ -89,11 +90,17 @@ class BuildEnvironment(object): if not reqs: return log.info('Calling pip to install %s', reqs) - check_call([sys.executable, '-m', 'pip', 'install', '--ignore-installed', - '--prefix', self.path] + list(reqs)) + check_call([ + sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--prefix', self.path] + list(reqs)) def __exit__(self, exc_type, exc_val, exc_tb): - if self._cleanup and (self.path is not None) and os.path.isdir(self.path): + needs_cleanup = ( + self._cleanup and + self.path is not None and + os.path.isdir(self.path) + ) + if needs_cleanup: shutil.rmtree(self.path) if self.save_path is None: @@ -106,6 +113,7 @@ class BuildEnvironment(object): else: os.environ['PYTHONPATH'] = self.save_pythonpath + def build_wheel(source_dir, wheel_dir, config_settings=None): """Build a wheel from a source directory using PEP 517 hooks. diff --git a/pipenv/patched/notpip/_vendor/pep517/wrappers.py b/pipenv/patched/notpip/_vendor/pep517/wrappers.py index 28260f32..b14b8991 100644 --- a/pipenv/patched/notpip/_vendor/pep517/wrappers.py +++ b/pipenv/patched/notpip/_vendor/pep517/wrappers.py @@ -10,6 +10,7 @@ from . import compat _in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.py') + @contextmanager def tempdir(): td = mkdtemp() @@ -18,9 +19,24 @@ def tempdir(): finally: shutil.rmtree(td) + +class BackendUnavailable(Exception): + """Will be raised if the backend cannot be imported in the hook process.""" + + class UnsupportedOperation(Exception): """May be raised by build_sdist if the backend indicates that it can't.""" + +def default_subprocess_runner(cmd, cwd=None, extra_environ=None): + """The default method of calling the wrapper subprocess.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + check_call(cmd, cwd=cwd, env=env) + + class Pep517HookCaller(object): """A wrapper around a source directory to be built with a PEP 517 backend. @@ -30,6 +46,16 @@ class Pep517HookCaller(object): def __init__(self, source_dir, build_backend): self.source_dir = abspath(source_dir) self.build_backend = build_backend + self._subprocess_runner = default_subprocess_runner + + # TODO: Is this over-engineered? Maybe frontends only need to + # set this when creating the wrapper, not on every call. + @contextmanager + def subprocess_runner(self, runner): + prev = self._subprocess_runner + self._subprocess_runner = runner + yield + self._subprocess_runner = prev def get_requires_for_build_wheel(self, config_settings=None): """Identify packages required for building a wheel @@ -45,7 +71,8 @@ class Pep517HookCaller(object): 'config_settings': config_settings }) - def prepare_metadata_for_build_wheel(self, metadata_directory, config_settings=None): + def prepare_metadata_for_build_wheel( + self, metadata_directory, config_settings=None): """Prepare a *.dist-info folder with metadata for this project. Returns the name of the newly created folder. @@ -59,7 +86,9 @@ class Pep517HookCaller(object): 'config_settings': config_settings, }) - def build_wheel(self, wheel_directory, config_settings=None, metadata_directory=None): + def build_wheel( + self, wheel_directory, config_settings=None, + metadata_directory=None): """Build a wheel from this project. Returns the name of the newly created file. @@ -103,10 +132,7 @@ class Pep517HookCaller(object): 'config_settings': config_settings, }) - def _call_hook(self, hook_name, kwargs): - env = os.environ.copy() - # On Python 2, pytoml returns Unicode values (which is correct) but the # environment passed to check_call needs to contain string values. We # convert here by encoding using ASCII (the backend can only contain @@ -118,17 +144,20 @@ class Pep517HookCaller(object): else: build_backend = self.build_backend - env['PEP517_BUILD_BACKEND'] = build_backend with tempdir() as td: compat.write_json({'kwargs': kwargs}, pjoin(td, 'input.json'), indent=2) # Run the hook in a subprocess - check_call([sys.executable, _in_proc_script, hook_name, td], - cwd=self.source_dir, env=env) + self._subprocess_runner( + [sys.executable, _in_proc_script, hook_name, td], + cwd=self.source_dir, + extra_environ={'PEP517_BUILD_BACKEND': build_backend} + ) data = compat.read_json(pjoin(td, 'output.json')) if data.get('unsupported'): raise UnsupportedOperation + if data.get('no_backend'): + raise BackendUnavailable return data['return_val'] - diff --git a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py index ac893b66..0459a7da 100644 --- a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py +++ b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py @@ -238,6 +238,9 @@ __all__ = [ 'register_finder', 'register_namespace_handler', 'register_loader_type', 'fixup_namespace_packages', 'get_importer', + # Warnings + 'PkgResourcesDeprecationWarning', + # Deprecated/backward compatibility only 'run_main', 'AvailableDistributions', ] @@ -2228,7 +2231,18 @@ register_namespace_handler(object, null_ns_handler) def normalize_path(filename): """Normalize a file/dir name for comparison purposes""" - return os.path.normcase(os.path.realpath(filename)) + return os.path.normcase(os.path.realpath(os.path.normpath(_cygwin_patch(filename)))) + + +def _cygwin_patch(filename): # pragma: nocover + """ + Contrary to POSIX 2008, on Cygwin, getcwd (3) contains + symlink components. Using + os.path.abspath() works around this limitation. A fix in os.getcwd() + would probably better, in Cygwin even more so, except + that this seems to be by design... + """ + return os.path.abspath(filename) if sys.platform == 'cygwin' else filename def _normalize_cached(filename, _cache={}): @@ -2324,7 +2338,7 @@ class EntryPoint: warnings.warn( "Parameters to load are deprecated. Call .resolve and " ".require separately.", - DeprecationWarning, + PkgResourcesDeprecationWarning, stacklevel=2, ) if require: @@ -3147,3 +3161,11 @@ def _initialize_master_working_set(): # match order list(map(working_set.add_entry, sys.path)) globals().update(locals()) + +class PkgResourcesDeprecationWarning(Warning): + """ + Base class for warning about deprecations in ``pkg_resources`` + + This class is not derived from ``DeprecationWarning``, and as such is + visible by default. + """ diff --git a/pipenv/patched/notpip/_vendor/pyparsing.py b/pipenv/patched/notpip/_vendor/pyparsing.py index 455d1151..3972b370 100644 --- a/pipenv/patched/notpip/_vendor/pyparsing.py +++ b/pipenv/patched/notpip/_vendor/pyparsing.py @@ -1,6 +1,7 @@ +#-*- coding: utf-8 -*- # module pyparsing.py # -# Copyright (c) 2003-2018 Paul T. McGuire +# Copyright (c) 2003-2019 Paul T. McGuire # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -27,15 +28,18 @@ __doc__ = \ pyparsing module - Classes and methods to define and execute parsing grammars ============================================================================= -The pyparsing module is an alternative approach to creating and executing simple grammars, -vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you -don't need to learn a new syntax for defining grammars or matching expressions - the parsing module -provides a library of classes that you use to construct the grammar directly in Python. +The pyparsing module is an alternative approach to creating and +executing simple grammars, vs. the traditional lex/yacc approach, or the +use of regular expressions. With pyparsing, you don't need to learn +a new syntax for defining grammars or matching expressions - the parsing +module provides a library of classes that you use to construct the +grammar directly in Python. -Here is a program to parse "Hello, World!" (or any greeting of the form -C{", !"}), built up using L{Word}, L{Literal}, and L{And} elements -(L{'+'} operator gives L{And} expressions, strings are auto-converted to -L{Literal} expressions):: +Here is a program to parse "Hello, World!" (or any greeting of the form +``", !"``), built up using :class:`Word`, +:class:`Literal`, and :class:`And` elements +(the :class:`'+'` operators create :class:`And` expressions, +and the strings are auto-converted to :class:`Literal` expressions):: from pipenv.patched.notpip._vendor.pyparsing import Word, alphas @@ -49,33 +53,48 @@ The program outputs the following:: Hello, World! -> ['Hello', ',', 'World', '!'] -The Python representation of the grammar is quite readable, owing to the self-explanatory -class names, and the use of '+', '|' and '^' operators. +The Python representation of the grammar is quite readable, owing to the +self-explanatory class names, and the use of '+', '|' and '^' operators. -The L{ParseResults} object returned from L{ParserElement.parseString} can be accessed as a nested list, a dictionary, or an -object with named attributes. +The :class:`ParseResults` object returned from +:class:`ParserElement.parseString` can be +accessed as a nested list, a dictionary, or an object with named +attributes. -The pyparsing module handles some of the problems that are typically vexing when writing text parsers: - - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) - - quoted strings - - embedded comments +The pyparsing module handles some of the problems that are typically +vexing when writing text parsers: + + - extra or missing whitespace (the above program will also handle + "Hello,World!", "Hello , World !", etc.) + - quoted strings + - embedded comments Getting Started - ----------------- -Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing +Visit the classes :class:`ParserElement` and :class:`ParseResults` to +see the base classes that most other pyparsing classes inherit from. Use the docstrings for examples of how to: - - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes - - construct character word-group expressions using the L{Word} class - - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes - - use L{'+'}, L{'|'}, L{'^'}, and L{'&'} operators to combine simple expressions into more complex ones - - associate names with your parsed results using L{ParserElement.setResultsName} - - find some helpful expression short-cuts like L{delimitedList} and L{oneOf} - - find more useful common expressions in the L{pyparsing_common} namespace class + + - construct literal match expressions from :class:`Literal` and + :class:`CaselessLiteral` classes + - construct character word-group expressions using the :class:`Word` + class + - see how to create repetitive expressions using :class:`ZeroOrMore` + and :class:`OneOrMore` classes + - use :class:`'+'`, :class:`'|'`, :class:`'^'`, + and :class:`'&'` operators to combine simple expressions into + more complex ones + - associate names with your parsed results using + :class:`ParserElement.setResultsName` + - find some helpful expression short-cuts like :class:`delimitedList` + and :class:`oneOf` + - find more useful common expressions in the :class:`pyparsing_common` + namespace class """ -__version__ = "2.2.1" -__versionTime__ = "18 Sep 2018 00:49 UTC" +__version__ = "2.3.1" +__versionTime__ = "09 Jan 2019 23:26 UTC" __author__ = "Paul McGuire " import string @@ -91,6 +110,12 @@ import traceback import types from datetime import datetime +try: + # Python 3 + from itertools import filterfalse +except ImportError: + from itertools import ifilterfalse as filterfalse + try: from _thread import RLock except ImportError: @@ -113,27 +138,33 @@ except ImportError: except ImportError: _OrderedDict = None +try: + from types import SimpleNamespace +except ImportError: + class SimpleNamespace: pass + + #~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) ) __all__ = [ 'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', 'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', -'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', +'PrecededBy', 'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', 'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', 'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', -'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', -'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', +'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', +'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', 'Char', 'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', 'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', 'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums', 'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno', 'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', 'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', -'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', +'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', 'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', 'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', 'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass', -'CloseMatch', 'tokenMap', 'pyparsing_common', +'CloseMatch', 'tokenMap', 'pyparsing_common', 'pyparsing_unicode', 'unicode_set', ] system_version = tuple(sys.version_info)[:3] @@ -142,6 +173,7 @@ if PY_3: _MAX_INT = sys.maxsize basestring = str unichr = chr + unicode = str _ustr = str # build list of single arg builtins, that can be used as parse actions @@ -152,9 +184,11 @@ else: range = xrange def _ustr(obj): - """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries - str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It - then < returns the unicode object | encodes it with the default encoding | ... >. + """Drop-in replacement for str(obj) that tries to be Unicode + friendly. It first tries str(obj). If that fails with + a UnicodeEncodeError, then it tries unicode(obj). It then + < returns the unicode object | encodes it with the default + encoding | ... >. """ if isinstance(obj,unicode): return obj @@ -179,9 +213,9 @@ else: singleArgBuiltins.append(getattr(__builtin__,fname)) except AttributeError: continue - + _generatorType = type((y for y in range(1))) - + def _xml_escape(data): """Escape &, <, >, ", ', etc. in a string of data.""" @@ -192,9 +226,6 @@ def _xml_escape(data): data = data.replace(from_, to_) return data -class _Constants(object): - pass - alphas = string.ascii_uppercase + string.ascii_lowercase nums = "0123456789" hexnums = nums + "ABCDEFabcdef" @@ -220,16 +251,16 @@ class ParseBaseException(Exception): @classmethod def _from_exception(cls, pe): """ - internal factory method to simplify creating one type of ParseException + internal factory method to simplify creating one type of ParseException from another - avoids having __init__ signature conflicts among subclasses """ return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement) def __getattr__( self, aname ): """supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text """ if( aname == "lineno" ): return lineno( self.loc, self.pstr ) @@ -262,22 +293,94 @@ class ParseException(ParseBaseException): """ Exception thrown when parse expressions don't match class; supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text - + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text + Example:: + try: Word(nums).setName("integer").parseString("ABC") except ParseException as pe: print(pe) print("column: {}".format(pe.col)) - + prints:: + Expected integer (at char 0), (line:1, col:1) column: 1 + """ - pass + + @staticmethod + def explain(exc, depth=16): + """ + Method to take an exception and translate the Python internal traceback into a list + of the pyparsing expressions that caused the exception to be raised. + + Parameters: + + - exc - exception raised during parsing (need not be a ParseException, in support + of Python exceptions that might be raised in a parse action) + - depth (default=16) - number of levels back in the stack trace to list expression + and function names; if None, the full stack trace names will be listed; if 0, only + the failing input line, marker, and exception string will be shown + + Returns a multi-line string listing the ParserElements and/or function names in the + exception's stack trace. + + Note: the diagnostic output will include string representations of the expressions + that failed to parse. These representations will be more helpful if you use `setName` to + give identifiable names to your expressions. Otherwise they will use the default string + forms, which may be cryptic to read. + + explain() is only supported under Python 3. + """ + import inspect + + if depth is None: + depth = sys.getrecursionlimit() + ret = [] + if isinstance(exc, ParseBaseException): + ret.append(exc.line) + ret.append(' ' * (exc.col - 1) + '^') + ret.append("{0}: {1}".format(type(exc).__name__, exc)) + + if depth > 0: + callers = inspect.getinnerframes(exc.__traceback__, context=depth) + seen = set() + for i, ff in enumerate(callers[-depth:]): + frm = ff.frame + + f_self = frm.f_locals.get('self', None) + if isinstance(f_self, ParserElement): + if frm.f_code.co_name not in ('parseImpl', '_parseNoCache'): + continue + if f_self in seen: + continue + seen.add(f_self) + + self_type = type(f_self) + ret.append("{0}.{1} - {2}".format(self_type.__module__, + self_type.__name__, + f_self)) + elif f_self is not None: + self_type = type(f_self) + ret.append("{0}.{1}".format(self_type.__module__, + self_type.__name__)) + else: + code = frm.f_code + if code.co_name in ('wrapper', ''): + continue + + ret.append("{0}".format(code.co_name)) + + depth -= 1 + if not depth: + break + + return '\n'.join(ret) + class ParseFatalException(ParseBaseException): """user-throwable exception thrown when inconsistent parse content @@ -285,9 +388,11 @@ class ParseFatalException(ParseBaseException): pass class ParseSyntaxException(ParseFatalException): - """just like L{ParseFatalException}, but thrown internally when an - L{ErrorStop} ('-' operator) indicates that parsing is to stop - immediately because an unbacktrackable syntax error has been found""" + """just like :class:`ParseFatalException`, but thrown internally + when an :class:`ErrorStop` ('-' operator) indicates + that parsing is to stop immediately because an unbacktrackable + syntax error has been found. + """ pass #~ class ReparseException(ParseBaseException): @@ -304,7 +409,9 @@ class ParseSyntaxException(ParseFatalException): #~ self.reparseLoc = restartLoc class RecursiveGrammarException(Exception): - """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive""" + """exception thrown by :class:`ParserElement.validate` if the + grammar could be improperly recursive + """ def __init__( self, parseElementList ): self.parseElementTrace = parseElementList @@ -322,16 +429,18 @@ class _ParseResultsWithOffset(object): self.tup = (self.tup[0],i) class ParseResults(object): - """ - Structured parse results, to provide multiple means of access to the parsed data: - - as a list (C{len(results)}) - - by list index (C{results[0], results[1]}, etc.) - - by attribute (C{results.} - see L{ParserElement.setResultsName}) + """Structured parse results, to provide multiple means of access to + the parsed data: + + - as a list (``len(results)``) + - by list index (``results[0], results[1]``, etc.) + - by attribute (``results.`` - see :class:`ParserElement.setResultsName`) Example:: + integer = Word(nums) - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: # date_str = integer("year") + '/' + integer("month") + '/' + integer("day") @@ -348,7 +457,9 @@ class ParseResults(object): test("'month' in result") test("'minutes' in result") test("result.dump()", str) + prints:: + list(result) -> ['1999', '/', '12', '/', '31'] result[0] -> '1999' result['month'] -> '12' @@ -398,7 +509,7 @@ class ParseResults(object): toklist = [ toklist ] if asList: if isinstance(toklist,ParseResults): - self[name] = _ParseResultsWithOffset(toklist.copy(),0) + self[name] = _ParseResultsWithOffset(ParseResults(toklist.__toklist), 0) else: self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0) self[name].__name = name @@ -467,19 +578,19 @@ class ParseResults(object): def _itervalues( self ): return (self[k] for k in self._iterkeys()) - + def _iteritems( self ): return ((k, self[k]) for k in self._iterkeys()) if PY_3: - keys = _iterkeys - """Returns an iterator of all named result keys (Python 3.x only).""" + keys = _iterkeys + """Returns an iterator of all named result keys.""" values = _itervalues - """Returns an iterator of all named result values (Python 3.x only).""" + """Returns an iterator of all named result values.""" items = _iteritems - """Returns an iterator of all named result key-value tuples (Python 3.x only).""" + """Returns an iterator of all named result key-value tuples.""" else: iterkeys = _iterkeys @@ -498,7 +609,7 @@ class ParseResults(object): def values( self ): """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x).""" return list(self.itervalues()) - + def items( self ): """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x).""" return list(self.iteritems()) @@ -507,19 +618,20 @@ class ParseResults(object): """Since keys() returns an iterator, this method is helpful in bypassing code that looks for the existence of any defined results names.""" return bool(self.__tokdict) - + def pop( self, *args, **kwargs): """ - Removes and returns item at specified index (default=C{last}). - Supports both C{list} and C{dict} semantics for C{pop()}. If passed no - argument or an integer argument, it will use C{list} semantics - and pop tokens from the list of parsed tokens. If passed a - non-integer argument (most likely a string), it will use C{dict} - semantics and pop the corresponding value from any defined - results names. A second default return value argument is - supported, just as in C{dict.pop()}. + Removes and returns item at specified index (default= ``last``). + Supports both ``list`` and ``dict`` semantics for ``pop()``. If + passed no argument or an integer argument, it will use ``list`` + semantics and pop tokens from the list of parsed tokens. If passed + a non-integer argument (most likely a string), it will use ``dict`` + semantics and pop the corresponding value from any defined results + names. A second default return value argument is supported, just as in + ``dict.pop()``. Example:: + def remove_first(tokens): tokens.pop(0) print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] @@ -536,7 +648,9 @@ class ParseResults(object): return tokens patt.addParseAction(remove_LABEL) print(patt.parseString("AAB 123 321").dump()) + prints:: + ['AAB', '123', '321'] - LABEL: AAB @@ -549,8 +663,8 @@ class ParseResults(object): args = (args[0], v) else: raise TypeError("pop() got an unexpected keyword argument '%s'" % k) - if (isinstance(args[0], int) or - len(args) == 1 or + if (isinstance(args[0], int) or + len(args) == 1 or args[0] in self): index = args[0] ret = self[index] @@ -563,14 +677,15 @@ class ParseResults(object): def get(self, key, defaultValue=None): """ Returns named result matching the given key, or if there is no - such name, then returns the given C{defaultValue} or C{None} if no - C{defaultValue} is specified. + such name, then returns the given ``defaultValue`` or ``None`` if no + ``defaultValue`` is specified. + + Similar to ``dict.get()``. - Similar to C{dict.get()}. - Example:: + integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") result = date_str.parseString("1999/12/31") print(result.get("year")) # -> '1999' @@ -585,10 +700,11 @@ class ParseResults(object): def insert( self, index, insStr ): """ Inserts new element at location index in the list of parsed tokens. - - Similar to C{list.insert()}. + + Similar to ``list.insert()``. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] # use a parse action to insert the parse location in the front of the parsed results @@ -607,8 +723,9 @@ class ParseResults(object): Add single element to end of ParseResults list of elements. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - + # use a parse action to compute the sum of the parsed integers, and add it to the end def append_sum(tokens): tokens.append(sum(map(int, tokens))) @@ -621,8 +738,9 @@ class ParseResults(object): Add sequence of elements to end of ParseResults list of elements. Example:: + patt = OneOrMore(Word(alphas)) - + # use a parse action to append the reverse of the matched strings, to make a palindrome def make_palindrome(tokens): tokens.extend(reversed([t[::-1] for t in tokens])) @@ -646,7 +764,7 @@ class ParseResults(object): return self[name] except KeyError: return "" - + if name in self.__tokdict: if name not in self.__accumNames: return self.__tokdict[name][-1][0] @@ -671,7 +789,7 @@ class ParseResults(object): self[k] = v if isinstance(v[0],ParseResults): v[0].__parent = wkref(self) - + self.__toklist += other.__toklist self.__accumNames.update( other.__accumNames ) return self @@ -683,7 +801,7 @@ class ParseResults(object): else: # this may raise a TypeError - so be it return other + self - + def __repr__( self ): return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) ) @@ -706,11 +824,12 @@ class ParseResults(object): Returns the parse results as a nested list of matching tokens, all converted to strings. Example:: + patt = OneOrMore(Word(alphas)) result = patt.parseString("sldkj lsdkj sldkj") # even though the result prints in string-like form, it is actually a pyparsing ParseResults print(type(result), result) # -> ['sldkj', 'lsdkj', 'sldkj'] - + # Use asList() to create an actual list result_list = result.asList() print(type(result_list), result_list) # -> ['sldkj', 'lsdkj', 'sldkj'] @@ -722,12 +841,13 @@ class ParseResults(object): Returns the named parse results as a nested dictionary. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(type(result), repr(result)) # -> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]}) - + result_dict = result.asDict() print(type(result_dict), repr(result_dict)) # -> {'day': '1999', 'year': '12', 'month': '31'} @@ -740,7 +860,7 @@ class ParseResults(object): item_fn = self.items else: item_fn = self.iteritems - + def toItem(obj): if isinstance(obj, ParseResults): if obj.haskeys(): @@ -749,15 +869,15 @@ class ParseResults(object): return [toItem(v) for v in obj] else: return obj - + return dict((k,toItem(v)) for k,v in item_fn()) def copy( self ): """ - Returns a new copy of a C{ParseResults} object. + Returns a new copy of a :class:`ParseResults` object. """ ret = ParseResults( self.__toklist ) - ret.__tokdict = self.__tokdict.copy() + ret.__tokdict = dict(self.__tokdict.items()) ret.__parent = self.__parent ret.__accumNames.update( self.__accumNames ) ret.__name = self.__name @@ -833,22 +953,25 @@ class ParseResults(object): def getName(self): r""" - Returns the results name for this token expression. Useful when several + Returns the results name for this token expression. Useful when several different expressions might match at a particular location. Example:: + integer = Word(nums) ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d") house_number_expr = Suppress('#') + Word(nums, alphanums) - user_data = (Group(house_number_expr)("house_number") + user_data = (Group(house_number_expr)("house_number") | Group(ssn_expr)("ssn") | Group(integer)("age")) user_info = OneOrMore(user_data) - + result = user_info.parseString("22 111-22-3333 #221B") for item in result: print(item.getName(), ':', item[0]) + prints:: + age : 22 ssn : 111-22-3333 house_number : 221B @@ -870,17 +993,20 @@ class ParseResults(object): def dump(self, indent='', depth=0, full=True): """ - Diagnostic method for listing out the contents of a C{ParseResults}. - Accepts an optional C{indent} argument so that this string can be embedded - in a nested display of other data. + Diagnostic method for listing out the contents of + a :class:`ParseResults`. Accepts an optional ``indent`` argument so + that this string can be embedded in a nested display of other data. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(result.dump()) + prints:: + ['12', '/', '31', '/', '1999'] - day: 1999 - month: 31 @@ -910,16 +1036,18 @@ class ParseResults(object): out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) )) else: out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv))) - + return "".join(out) def pprint(self, *args, **kwargs): """ - Pretty-printer for parsed results as a list, using the C{pprint} module. - Accepts additional positional or keyword args as defined for the - C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint}) + Pretty-printer for parsed results as a list, using the + `pprint `_ module. + Accepts additional positional or keyword args as defined for + `pprint.pprint `_ . Example:: + ident = Word(alphas, alphanums) num = Word(nums) func = Forward() @@ -927,7 +1055,9 @@ class ParseResults(object): func <<= ident + Group(Optional(delimitedList(term))) result = func.parseString("fna a,b,(fnb c,d,200),100") result.pprint(width=40) + prints:: + ['fna', ['a', 'b', @@ -970,24 +1100,25 @@ def col (loc,strg): The first column is number 1. Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{ParserElement.parseString}} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. + before starting the parsing process. See + :class:`ParserElement.parseString` for more + information on parsing strings containing ```` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. """ s = strg return 1 if 0} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - """ + Note - the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See :class:`ParserElement.parseString` + for more information on parsing strings containing ```` s, and + suggested methods to maintain a consistent view of the parsed string, the + parse location, and line and column positions within the parsed string. + """ return strg.count("\n",0,loc) + 1 def line( loc, strg ): @@ -1041,7 +1172,7 @@ def _trim_arity(func, maxargs=2): return lambda s,l,t: func(t) limit = [0] foundArity = [False] - + # traceback return data structure changed in Py3.5 - normalize back to plain tuples if system_version[:2] >= (3,5): def extract_stack(limit=0): @@ -1056,12 +1187,12 @@ def _trim_arity(func, maxargs=2): else: extract_stack = traceback.extract_stack extract_tb = traceback.extract_tb - - # synthesize what would be returned by traceback.extract_stack at the call to + + # synthesize what would be returned by traceback.extract_stack at the call to # user's parse action 'func', so that we don't incur call penalty at parse time - + LINE_DIFF = 6 - # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND + # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!! this_line = extract_stack(limit=2)[-1] pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF) @@ -1092,7 +1223,7 @@ def _trim_arity(func, maxargs=2): # copy func name to wrapper for sensible debug output func_name = "" try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -1111,9 +1242,10 @@ class ParserElement(object): Overrides the default whitespace chars Example:: + # default whitespace chars are space, and newline OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl'] - + # change to just treat newline as significant ParserElement.setDefaultWhitespaceChars(" \t") OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def'] @@ -1124,18 +1256,19 @@ class ParserElement(object): def inlineLiteralsUsing(cls): """ Set class to be used for inclusion of string literals into a parser. - + Example:: + # default literal class used is Literal integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31'] # change to Suppress ParserElement.inlineLiteralsUsing(Suppress) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '12', '31'] """ @@ -1149,7 +1282,7 @@ class ParserElement(object): self.resultsName = None self.saveAsList = savelist self.skipWhitespace = True - self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS + self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) self.copyDefaultWhiteChars = True self.mayReturnEmpty = False # used when checking for left-recursion self.keepTabs = False @@ -1166,18 +1299,24 @@ class ParserElement(object): def copy( self ): """ - Make a copy of this C{ParserElement}. Useful for defining different parse actions - for the same parsing pattern, using copies of the original parse element. - + Make a copy of this :class:`ParserElement`. Useful for defining + different parse actions for the same parsing pattern, using copies of + the original parse element. + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K") integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") - + print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M")) + prints:: + [5120, 100, 655360, 268435456] - Equivalent form of C{expr.copy()} is just C{expr()}:: + + Equivalent form of ``expr.copy()`` is just ``expr()``:: + integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") """ cpy = copy.copy( self ) @@ -1190,8 +1329,9 @@ class ParserElement(object): def setName( self, name ): """ Define name for this expression, makes debugging and exception messages clearer. - + Example:: + Word(nums).parseString("ABC") # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1) Word(nums).setName("integer").parseString("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1) """ @@ -1205,17 +1345,18 @@ class ParserElement(object): """ Define name for referencing matching tokens as a nested attribute of the returned parse results. - NOTE: this returns a *copy* of the original C{ParserElement} object; + NOTE: this returns a *copy* of the original :class:`ParserElement` object; this is so that the client can define a basic element, such as an integer, and reference it in multiple places with different names. You can also set results names using the abbreviated syntax, - C{expr("name")} in place of C{expr.setResultsName("name")} - - see L{I{__call__}<__call__>}. + ``expr("name")`` in place of ``expr.setResultsName("name")`` + - see :class:`__call__`. Example:: - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: @@ -1231,7 +1372,7 @@ class ParserElement(object): def setBreak(self,breakFlag = True): """Method to invoke the Python pdb debugger when this element is - about to be parsed. Set C{breakFlag} to True to enable, False to + about to be parsed. Set ``breakFlag`` to True to enable, False to disable. """ if breakFlag: @@ -1250,25 +1391,28 @@ class ParserElement(object): def setParseAction( self, *fns, **kwargs ): """ Define one or more actions to perform when successfully matching parse element definition. - Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)}, - C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where: - - s = the original string being parsed (see note below) - - loc = the location of the matching substring - - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object + Parse action fn is a callable method with 0-3 arguments, called as ``fn(s,loc,toks)`` , + ``fn(loc,toks)`` , ``fn(toks)`` , or just ``fn()`` , where: + + - s = the original string being parsed (see note below) + - loc = the location of the matching substring + - toks = a list of the matched tokens, packaged as a :class:`ParseResults` object + If the functions in fns modify the tokens, they can return them as the return value from fn, and the modified list of tokens will replace the original. Otherwise, fn does not need to return any value. Optional keyword arguments: - - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing + - callDuringTry = (default= ``False`` ) indicate if parse action should be run during lookaheads and alternate testing Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{parseString}} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - + before starting the parsing process. See :class:`parseString for more + information on parsing strings containing ```` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. + Example:: + integer = Word(nums) date_str = integer + '/' + integer + '/' + integer @@ -1287,24 +1431,25 @@ class ParserElement(object): def addParseAction( self, *fns, **kwargs ): """ - Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}}. - - See examples in L{I{copy}}. + Add one or more parse actions to expression's list of parse actions. See :class:`setParseAction`. + + See examples in :class:`copy`. """ self.parseAction += list(map(_trim_arity, list(fns))) self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) return self def addCondition(self, *fns, **kwargs): - """Add a boolean predicate function to expression's list of parse actions. See - L{I{setParseAction}} for function call signatures. Unlike C{setParseAction}, - functions passed to C{addCondition} need to return boolean success/fail of the condition. + """Add a boolean predicate function to expression's list of parse actions. See + :class:`setParseAction` for function call signatures. Unlike ``setParseAction``, + functions passed to ``addCondition`` need to return boolean success/fail of the condition. Optional keyword arguments: - - message = define a custom message to be used in the raised exception - - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException - + - message = define a custom message to be used in the raised exception + - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) year_int = integer.copy() year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later") @@ -1315,8 +1460,9 @@ class ParserElement(object): msg = kwargs.get("message", "failed user-defined condition") exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException for fn in fns: + fn = _trim_arity(fn) def pa(s,l,t): - if not bool(_trim_arity(fn)(s,l,t)): + if not bool(fn(s,l,t)): raise exc_type(s,l,msg) self.parseAction.append(pa) self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) @@ -1325,12 +1471,12 @@ class ParserElement(object): def setFailAction( self, fn ): """Define action to perform if parsing fails at this expression. Fail acton fn is a callable function that takes the arguments - C{fn(s,loc,expr,err)} where: - - s = string being parsed - - loc = location where expression match was attempted and failed - - expr = the parse expression that failed - - err = the exception thrown - The function returns no value. It may throw C{L{ParseFatalException}} + ``fn(s,loc,expr,err)`` where: + - s = string being parsed + - loc = location where expression match was attempted and failed + - expr = the parse expression that failed + - err = the exception thrown + The function returns no value. It may throw :class:`ParseFatalException` if it is desired to stop parsing immediately.""" self.failAction = fn return self @@ -1412,8 +1558,14 @@ class ParserElement(object): if debugging: try: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: + try: + tokens = fn( instring, tokensStart, retTokens ) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), @@ -1425,8 +1577,14 @@ class ParserElement(object): raise else: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: + try: + tokens = fn( instring, tokensStart, retTokens ) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), @@ -1443,7 +1601,7 @@ class ParserElement(object): return self._parse( instring, loc, doActions=False )[0] except ParseFatalException: raise ParseException( instring, loc, self.errmsg, self) - + def canParseNext(self, instring, loc): try: self.tryParse(instring, loc) @@ -1465,7 +1623,7 @@ class ParserElement(object): def clear(self): cache.clear() - + def cache_len(self): return len(cache) @@ -1577,23 +1735,23 @@ class ParserElement(object): often in many complex grammars) can immediately return a cached value, instead of re-executing parsing/validating code. Memoizing is done of both valid results and parsing exceptions. - + Parameters: - - cache_size_limit - (default=C{128}) - if an integer value is provided - will limit the size of the packrat cache; if None is passed, then - the cache size will be unbounded; if 0 is passed, the cache will - be effectively disabled. - + + - cache_size_limit - (default= ``128``) - if an integer value is provided + will limit the size of the packrat cache; if None is passed, then + the cache size will be unbounded; if 0 is passed, the cache will + be effectively disabled. + This speedup may break existing programs that use parse actions that have side-effects. For this reason, packrat parsing is disabled when you first import pyparsing. To activate the packrat feature, your - program must call the class method C{ParserElement.enablePackrat()}. If - your program uses C{psyco} to "compile as you go", you must call - C{enablePackrat} before calling C{psyco.full()}. If you do not do this, - Python will crash. For best results, call C{enablePackrat()} immediately - after importing pyparsing. - + program must call the class method :class:`ParserElement.enablePackrat`. + For best results, call ``enablePackrat()`` immediately after + importing pyparsing. + Example:: + from pipenv.patched.notpip._vendor import pyparsing pyparsing.ParserElement.enablePackrat() """ @@ -1612,23 +1770,25 @@ class ParserElement(object): expression has been built. If you want the grammar to require that the entire input string be - successfully parsed, then set C{parseAll} to True (equivalent to ending - the grammar with C{L{StringEnd()}}). + successfully parsed, then set ``parseAll`` to True (equivalent to ending + the grammar with ``StringEnd()``). - Note: C{parseString} implicitly calls C{expandtabs()} on the input string, + Note: ``parseString`` implicitly calls ``expandtabs()`` on the input string, in order to report proper column numbers in parse actions. If the input string contains tabs and - the grammar uses parse actions that use the C{loc} argument to index into the + the grammar uses parse actions that use the ``loc`` argument to index into the string being parsed, you can ensure you have a consistent view of the input string by: - - calling C{parseWithTabs} on your grammar before calling C{parseString} - (see L{I{parseWithTabs}}) - - define your parse action using the full C{(s,loc,toks)} signature, and - reference the input string using the parse action's C{s} argument - - explictly expand the tabs in your input string before calling - C{parseString} - + + - calling ``parseWithTabs`` on your grammar before calling ``parseString`` + (see :class:`parseWithTabs`) + - define your parse action using the full ``(s,loc,toks)`` signature, and + reference the input string using the parse action's ``s`` argument + - explictly expand the tabs in your input string before calling + ``parseString`` + Example:: + Word('a').parseString('aaaaabaaa') # -> ['aaaaa'] Word('a').parseString('aaaaabaaa', parseAll=True) # -> Exception: Expected end of text """ @@ -1659,22 +1819,23 @@ class ParserElement(object): """ Scan the input string for expression matches. Each match will return the matching tokens, start location, and end location. May be called with optional - C{maxMatches} argument, to clip scanning after 'n' matches are found. If - C{overlap} is specified, then overlapping matches will be reported. + ``maxMatches`` argument, to clip scanning after 'n' matches are found. If + ``overlap`` is specified, then overlapping matches will be reported. Note that the start and end locations are reported relative to the string - being parsed. See L{I{parseString}} for more information on parsing + being parsed. See :class:`parseString` for more information on parsing strings with embedded tabs. Example:: + source = "sldjf123lsdjjkf345sldkjf879lkjsfd987" print(source) for tokens,start,end in Word(alphas).scanString(source): print(' '*start + '^'*(end-start)) print(' '*start + tokens[0]) - + prints:: - + sldjf123lsdjjkf345sldkjf879lkjsfd987 ^^^^^ sldjf @@ -1728,19 +1889,22 @@ class ParserElement(object): def transformString( self, instring ): """ - Extension to C{L{scanString}}, to modify matching text with modified tokens that may - be returned from a parse action. To use C{transformString}, define a grammar and + Extension to :class:`scanString`, to modify matching text with modified tokens that may + be returned from a parse action. To use ``transformString``, define a grammar and attach a parse action to it that modifies the returned token list. - Invoking C{transformString()} on a target string will then scan for matches, + Invoking ``transformString()`` on a target string will then scan for matches, and replace the matched text patterns according to the logic in the parse - action. C{transformString()} returns the resulting transformed string. - + action. ``transformString()`` returns the resulting transformed string. + Example:: + wd = Word(alphas) wd.setParseAction(lambda toks: toks[0].title()) - + print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york.")) - Prints:: + + prints:: + Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York. """ out = [] @@ -1771,19 +1935,22 @@ class ParserElement(object): def searchString( self, instring, maxMatches=_MAX_INT ): """ - Another extension to C{L{scanString}}, simplifying the access to the tokens found + Another extension to :class:`scanString`, simplifying the access to the tokens found to match the given parse expression. May be called with optional - C{maxMatches} argument, to clip searching after 'n' matches are found. - + ``maxMatches`` argument, to clip searching after 'n' matches are found. + Example:: + # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters cap_word = Word(alphas.upper(), alphas.lower()) - + print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")) # the sum() builtin can be used to merge results into a single ParseResults object print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))) + prints:: + [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']] ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity'] """ @@ -1799,14 +1966,17 @@ class ParserElement(object): def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False): """ Generator method to split a string using the given expression as a separator. - May be called with optional C{maxsplit} argument, to limit the number of splits; - and the optional C{includeSeparators} argument (default=C{False}), if the separating + May be called with optional ``maxsplit`` argument, to limit the number of splits; + and the optional ``includeSeparators`` argument (default= ``False``), if the separating matching text should be included in the split results. - - Example:: + + Example:: + punc = oneOf(list(".,;:/-!?")) print(list(punc.split("This, this?, this sentence, is badly punctuated!"))) + prints:: + ['This', ' this', '', ' this sentence', ' is badly punctuated', ''] """ splits = 0 @@ -1820,14 +1990,17 @@ class ParserElement(object): def __add__(self, other ): """ - Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement - converts them to L{Literal}s by default. - + Implementation of + operator - returns :class:`And`. Adding strings to a ParserElement + converts them to :class:`Literal`s by default. + Example:: + greet = Word(alphas) + "," + Word(alphas) + "!" hello = "Hello, World!" print (hello, "->", greet.parseString(hello)) - Prints:: + + prints:: + Hello, World! -> ['Hello', ',', 'World', '!'] """ if isinstance( other, basestring ): @@ -1840,7 +2013,7 @@ class ParserElement(object): def __radd__(self, other ): """ - Implementation of + operator when left operand is not a C{L{ParserElement}} + Implementation of + operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1852,7 +2025,7 @@ class ParserElement(object): def __sub__(self, other): """ - Implementation of - operator, returns C{L{And}} with error stop + Implementation of - operator, returns :class:`And` with error stop """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1864,7 +2037,7 @@ class ParserElement(object): def __rsub__(self, other ): """ - Implementation of - operator when left operand is not a C{L{ParserElement}} + Implementation of - operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1876,23 +2049,23 @@ class ParserElement(object): def __mul__(self,other): """ - Implementation of * operator, allows use of C{expr * 3} in place of - C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer - tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples - may also include C{None} as in: - - C{expr*(n,None)} or C{expr*(n,)} is equivalent - to C{expr*n + L{ZeroOrMore}(expr)} - (read as "at least n instances of C{expr}") - - C{expr*(None,n)} is equivalent to C{expr*(0,n)} - (read as "0 to n instances of C{expr}") - - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)} - - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)} + Implementation of * operator, allows use of ``expr * 3`` in place of + ``expr + expr + expr``. Expressions may also me multiplied by a 2-integer + tuple, similar to ``{min,max}`` multipliers in regular expressions. Tuples + may also include ``None`` as in: + - ``expr*(n,None)`` or ``expr*(n,)`` is equivalent + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") + - ``expr*(None,n)`` is equivalent to ``expr*(0,n)`` + (read as "0 to n instances of ``expr``") + - ``expr*(None,None)`` is equivalent to ``ZeroOrMore(expr)`` + - ``expr*(1,None)`` is equivalent to ``OneOrMore(expr)`` - Note that C{expr*(None,n)} does not raise an exception if + Note that ``expr*(None,n)`` does not raise an exception if more than n exprs exist in the input stream; that is, - C{expr*(None,n)} does not enforce a maximum number of expr + ``expr*(None,n)`` does not enforce a maximum number of expr occurrences. If this behavior is desired, then write - C{expr*(None,n) + ~expr} + ``expr*(None,n) + ~expr`` """ if isinstance(other,int): minElements, optElements = other,0 @@ -1947,7 +2120,7 @@ class ParserElement(object): def __or__(self, other ): """ - Implementation of | operator - returns C{L{MatchFirst}} + Implementation of | operator - returns :class:`MatchFirst` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1959,7 +2132,7 @@ class ParserElement(object): def __ror__(self, other ): """ - Implementation of | operator when left operand is not a C{L{ParserElement}} + Implementation of | operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1971,7 +2144,7 @@ class ParserElement(object): def __xor__(self, other ): """ - Implementation of ^ operator - returns C{L{Or}} + Implementation of ^ operator - returns :class:`Or` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1983,7 +2156,7 @@ class ParserElement(object): def __rxor__(self, other ): """ - Implementation of ^ operator when left operand is not a C{L{ParserElement}} + Implementation of ^ operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1995,7 +2168,7 @@ class ParserElement(object): def __and__(self, other ): """ - Implementation of & operator - returns C{L{Each}} + Implementation of & operator - returns :class:`Each` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -2007,7 +2180,7 @@ class ParserElement(object): def __rand__(self, other ): """ - Implementation of & operator when left operand is not a C{L{ParserElement}} + Implementation of & operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -2019,23 +2192,24 @@ class ParserElement(object): def __invert__( self ): """ - Implementation of ~ operator - returns C{L{NotAny}} + Implementation of ~ operator - returns :class:`NotAny` """ return NotAny( self ) def __call__(self, name=None): """ - Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}. - - If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be - passed as C{True}. - - If C{name} is omitted, same as calling C{L{copy}}. + Shortcut for :class:`setResultsName`, with ``listAllMatches=False``. + + If ``name`` is given with a trailing ``'*'`` character, then ``listAllMatches`` will be + passed as ``True``. + + If ``name` is omitted, same as calling :class:`copy`. Example:: + # these are equivalent userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno") - userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") + userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") """ if name is not None: return self.setResultsName(name) @@ -2044,7 +2218,7 @@ class ParserElement(object): def suppress( self ): """ - Suppresses the output of this C{ParserElement}; useful to keep punctuation from + Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from cluttering up returned output. """ return Suppress( self ) @@ -2052,7 +2226,7 @@ class ParserElement(object): def leaveWhitespace( self ): """ Disables the skipping of whitespace before matching the characters in the - C{ParserElement}'s defined pattern. This is normally only used internally by + :class:`ParserElement`'s defined pattern. This is normally only used internally by the pyparsing module, but may be needed in some whitespace-sensitive grammars. """ self.skipWhitespace = False @@ -2069,9 +2243,9 @@ class ParserElement(object): def parseWithTabs( self ): """ - Overrides default behavior to expand C{}s to spaces before parsing the input string. - Must be called before C{parseString} when the input grammar contains elements that - match C{} characters. + Overrides default behavior to expand ````s to spaces before parsing the input string. + Must be called before ``parseString`` when the input grammar contains elements that + match ```` characters. """ self.keepTabs = True return self @@ -2081,11 +2255,12 @@ class ParserElement(object): Define expression to be ignored (e.g., comments) while doing pattern matching; may be called repeatedly, to define multiple comment or other ignorable patterns. - + Example:: + patt = OneOrMore(Word(alphas)) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj'] - + patt.ignore(cStyleComment) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd'] """ @@ -2112,19 +2287,21 @@ class ParserElement(object): def setDebug( self, flag=True ): """ Enable display of debugging messages while doing pattern matching. - Set C{flag} to True to enable, False to disable. + Set ``flag`` to True to enable, False to disable. Example:: + wd = Word(alphas).setName("alphaword") integer = Word(nums).setName("numword") term = wd | integer - + # turn on debugging for wd wd.setDebug() OneOrMore(term).parseString("abc 123 xyz 890") - + prints:: + Match alphaword at loc 0(1,1) Matched alphaword -> ['abc'] Match alphaword at loc 3(1,4) @@ -2137,12 +2314,12 @@ class ParserElement(object): Exception raised:Expected alphaword (at char 15), (line:1, col:16) The output shown is that produced by the default debug actions - custom debug actions can be - specified using L{setDebugActions}. Prior to attempting - to match the C{wd} expression, the debugging message C{"Match at loc (,)"} - is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"} - message is shown. Also note the use of L{setName} to assign a human-readable name to the expression, + specified using :class:`setDebugActions`. Prior to attempting + to match the ``wd`` expression, the debugging message ``"Match at loc (,)"`` + is shown. Then if the parse succeeds, a ``"Matched"`` message is shown, or an ``"Exception raised"`` + message is shown. Also note the use of :class:`setName` to assign a human-readable name to the expression, which makes debugging and exception messages easier to understand - for instance, the default - name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}. + name created for the :class:`Word` expression without calling ``setName`` is ``"W:(ABCD...)"``. """ if flag: self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction ) @@ -2212,14 +2389,15 @@ class ParserElement(object): def matches(self, testString, parseAll=True): """ - Method for quick testing of a parser against a test string. Good for simple + Method for quick testing of a parser against a test string. Good for simple inline microtests of sub expressions while building up larger parser. - + Parameters: - testString - to test against this expression for a match - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + Example:: + expr = Word(nums) assert expr.matches("100") """ @@ -2228,28 +2406,32 @@ class ParserElement(object): return True except ParseBaseException: return False - - def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False): + + def runTests(self, tests, parseAll=True, comment='#', + fullDump=True, printResults=True, failureTests=False, postParse=None): """ Execute the parse expression on a series of test strings, showing each test, the parsed results or where the parse failed. Quick and easy way to run a parse expression against a list of sample strings. - + Parameters: - tests - a list of separate test strings, or a multiline string of test strings - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - - comment - (default=C{'#'}) - expression for indicating embedded comments in the test + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + - comment - (default= ``'#'``) - expression for indicating embedded comments in the test string; pass None to disable comment filtering - - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline; + - fullDump - (default= ``True``) - dump results as list followed by results names in nested outline; if False, only dump nested list - - printResults - (default=C{True}) prints test output to stdout - - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing + - printResults - (default= ``True``) prints test output to stdout + - failureTests - (default= ``False``) indicates if these tests are expected to fail parsing + - postParse - (default= ``None``) optional callback for successful parse results; called as + `fn(test_string, parse_results)` and returns a string to be added to the test output Returns: a (success, results) tuple, where success indicates that all tests succeeded - (or failed if C{failureTests} is True), and the results contain a list of lines of each + (or failed if ``failureTests`` is True), and the results contain a list of lines of each test's output - + Example:: + number_expr = pyparsing_common.number.copy() result = number_expr.runTests(''' @@ -2273,7 +2455,9 @@ class ParserElement(object): 3.14.159 ''', failureTests=True) print("Success" if result[0] else "Failed!") + prints:: + # unsigned integer 100 [100] @@ -2291,7 +2475,7 @@ class ParserElement(object): [1e-12] Success - + # stray character 100Z ^ @@ -2313,7 +2497,7 @@ class ParserElement(object): lines, create a test like this:: expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines") - + (Note that this is a raw string literal, you must include the leading 'r'.) """ if isinstance(tests, basestring): @@ -2332,10 +2516,18 @@ class ParserElement(object): out = ['\n'.join(comments), t] comments = [] try: - t = t.replace(r'\n','\n') + # convert newline marks to actual newlines, and strip leading BOM if present + t = t.replace(r'\n','\n').lstrip('\ufeff') result = self.parseString(t, parseAll=parseAll) out.append(result.dump(full=fullDump)) success = success and not failureTests + if postParse is not None: + try: + pp_value = postParse(t, result) + if pp_value is not None: + out.append(str(pp_value)) + except Exception as e: + out.append("{0} failed: {1}: {2}".format(postParse.__name__, type(e).__name__, e)) except ParseBaseException as pe: fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else "" if '\n' in t: @@ -2357,21 +2549,20 @@ class ParserElement(object): print('\n'.join(out)) allResults.append((t, result)) - + return success, allResults - + class Token(ParserElement): - """ - Abstract C{ParserElement} subclass, for defining atomic matching patterns. + """Abstract :class:`ParserElement` subclass, for defining atomic + matching patterns. """ def __init__( self ): super(Token,self).__init__( savelist=False ) class Empty(Token): - """ - An empty token, will always match. + """An empty token, will always match. """ def __init__( self ): super(Empty,self).__init__() @@ -2381,8 +2572,7 @@ class Empty(Token): class NoMatch(Token): - """ - A token that will never match. + """A token that will never match. """ def __init__( self ): super(NoMatch,self).__init__() @@ -2396,18 +2586,18 @@ class NoMatch(Token): class Literal(Token): - """ - Token to exactly match a specified string. - + """Token to exactly match a specified string. + Example:: + Literal('blah').parseString('blah') # -> ['blah'] Literal('blah').parseString('blahfooblah') # -> ['blah'] Literal('blah').parseString('bla') # -> Exception: Expected "blah" - - For case-insensitive matching, use L{CaselessLiteral}. - + + For case-insensitive matching, use :class:`CaselessLiteral`. + For keyword matching (force word break before and after the matched string), - use L{Keyword} or L{CaselessKeyword}. + use :class:`Keyword` or :class:`CaselessKeyword`. """ def __init__( self, matchString ): super(Literal,self).__init__() @@ -2437,21 +2627,29 @@ _L = Literal ParserElement._literalStringClass = Literal class Keyword(Token): - """ - Token to exactly match a specified string as a keyword, that is, it must be - immediately followed by a non-keyword character. Compare with C{L{Literal}}: - - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}. - - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'} - Accepts two optional constructor arguments in addition to the keyword string: - - C{identChars} is a string of characters that would be valid identifier characters, - defaulting to all alphanumerics + "_" and "$" - - C{caseless} allows case-insensitive matching, default is C{False}. - + """Token to exactly match a specified string as a keyword, that is, + it must be immediately followed by a non-keyword character. Compare + with :class:`Literal`: + + - ``Literal("if")`` will match the leading ``'if'`` in + ``'ifAndOnlyIf'``. + - ``Keyword("if")`` will not; it will only match the leading + ``'if'`` in ``'if x=1'``, or ``'if(y==2)'`` + + Accepts two optional constructor arguments in addition to the + keyword string: + + - ``identChars`` is a string of characters that would be valid + identifier characters, defaulting to all alphanumerics + "_" and + "$" + - ``caseless`` allows case-insensitive matching, default is ``False``. + Example:: + Keyword("start").parseString("start") # -> ['start'] Keyword("start").parseString("starting") # -> Exception - For case-insensitive matching, use L{CaselessKeyword}. + For case-insensitive matching, use :class:`CaselessKeyword`. """ DEFAULT_KEYWORD_CHARS = alphanums+"_$" @@ -2502,15 +2700,15 @@ class Keyword(Token): Keyword.DEFAULT_KEYWORD_CHARS = chars class CaselessLiteral(Literal): - """ - Token to match a specified string, ignoring case of letters. + """Token to match a specified string, ignoring case of letters. Note: the matched results will always be in the case of the given match string, NOT the case of the input text. Example:: + OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD'] - - (Contrast with example for L{CaselessKeyword}.) + + (Contrast with example for :class:`CaselessKeyword`.) """ def __init__( self, matchString ): super(CaselessLiteral,self).__init__( matchString.upper() ) @@ -2526,36 +2724,39 @@ class CaselessLiteral(Literal): class CaselessKeyword(Keyword): """ - Caseless version of L{Keyword}. + Caseless version of :class:`Keyword`. Example:: + OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD'] - - (Contrast with example for L{CaselessLiteral}.) + + (Contrast with example for :class:`CaselessLiteral`.) """ def __init__( self, matchString, identChars=None ): super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True ) - def parseImpl( self, instring, loc, doActions=True ): - if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ): - return loc+self.matchLen, self.match - raise ParseException(instring, loc, self.errmsg, self) - class CloseMatch(Token): - """ - A variation on L{Literal} which matches "close" matches, that is, - strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters: - - C{match_string} - string to be matched - - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match - - The results from a successful parse will contain the matched text from the input string and the following named results: - - C{mismatches} - a list of the positions within the match_string where mismatches were found - - C{original} - the original match_string used to compare against the input string - - If C{mismatches} is an empty list, then the match was an exact match. - + """A variation on :class:`Literal` which matches "close" matches, + that is, strings with at most 'n' mismatching characters. + :class:`CloseMatch` takes parameters: + + - ``match_string`` - string to be matched + - ``maxMismatches`` - (``default=1``) maximum number of + mismatches allowed to count as a match + + The results from a successful parse will contain the matched text + from the input string and the following named results: + + - ``mismatches`` - a list of the positions within the + match_string where mismatches were found + - ``original`` - the original match_string used to compare + against the input string + + If ``mismatches`` is an empty list, then the match was an exact + match. + Example:: + patt = CloseMatch("ATCATCGAATGGA") patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']}) patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1) @@ -2604,49 +2805,55 @@ class CloseMatch(Token): class Word(Token): - """ - Token for matching words composed of allowed character sets. - Defined with string containing all allowed initial characters, - an optional string containing allowed body characters (if omitted, + """Token for matching words composed of allowed character sets. + Defined with string containing all allowed initial characters, an + optional string containing allowed body characters (if omitted, defaults to the initial character set), and an optional minimum, - maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. An optional - C{excludeChars} parameter can list characters that might be found in - the input C{bodyChars} string; useful to define a word of all printables - except for one or two characters, for instance. - - L{srange} is useful for defining custom character set strings for defining - C{Word} expressions, using range notation from regular expression character sets. - - A common mistake is to use C{Word} to match a specific literal string, as in - C{Word("Address")}. Remember that C{Word} uses the string argument to define - I{sets} of matchable characters. This expression would match "Add", "AAA", - "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'. - To match an exact literal string, use L{Literal} or L{Keyword}. + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. An optional ``excludeChars`` parameter can + list characters that might be found in the input ``bodyChars`` + string; useful to define a word of all printables except for one or + two characters, for instance. + + :class:`srange` is useful for defining custom character set strings + for defining ``Word`` expressions, using range notation from + regular expression character sets. + + A common mistake is to use :class:`Word` to match a specific literal + string, as in ``Word("Address")``. Remember that :class:`Word` + uses the string argument to define *sets* of matchable characters. + This expression would match "Add", "AAA", "dAred", or any other word + made up of the characters 'A', 'd', 'r', 'e', and 's'. To match an + exact literal string, use :class:`Literal` or :class:`Keyword`. pyparsing includes helper strings for building Words: - - L{alphas} - - L{nums} - - L{alphanums} - - L{hexnums} - - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.) - - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.) - - L{printables} (any non-whitespace character) + + - :class:`alphas` + - :class:`nums` + - :class:`alphanums` + - :class:`hexnums` + - :class:`alphas8bit` (alphabetic characters in ASCII range 128-255 + - accented, tilded, umlauted, etc.) + - :class:`punc8bit` (non-alphabetic characters in ASCII range + 128-255 - currency, symbols, superscripts, diacriticals, etc.) + - :class:`printables` (any non-whitespace character) Example:: + # a word composed of digits integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9")) - + # a word with a leading capital, and zero or more lowercase capital_word = Word(alphas.upper(), alphas.lower()) # hostnames are alphanumeric, with leading alpha, and '-' hostname = Word(alphas, alphanums+'-') - + # roman numeral (not a strict parser, accepts invalid mix of characters) roman = Word("IVXLCDM") - + # any string of non-whitespace characters, except for ',' csv_value = Word(printables, excludeChars=",") """ @@ -2762,22 +2969,38 @@ class Word(Token): return self.strRepr +class Char(Word): + """A short-cut class for defining ``Word(characters, exact=1)``, + when defining a match of any single character in a string of + characters. + """ + def __init__(self, charset): + super(Char, self).__init__(charset, exact=1) + self.reString = "[%s]" % _escapeRegexRangeChars(self.initCharsOrig) + self.re = re.compile( self.reString ) + + class Regex(Token): - r""" - Token for matching strings that match a given regular expression. - Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module. - If the given regex contains named groups (defined using C{(?P...)}), these will be preserved as - named parse results. + r"""Token for matching strings that match a given regular + expression. Defined with string specifying the regular expression in + a form recognized by the stdlib Python `re module `_. + If the given regex contains named groups (defined using ``(?P...)``), + these will be preserved as named parse results. Example:: + realnum = Regex(r"[+-]?\d+\.\d*") date = Regex(r'(?P\d{4})-(?P\d\d?)-(?P\d\d?)') - # ref: http://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression - roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") + # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression + roman = Regex(r"M{0,4}(CM|CD|D?{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") """ compiledREtype = type(re.compile("[A-Z]")) - def __init__( self, pattern, flags=0): - """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags.""" + def __init__( self, pattern, flags=0, asGroupList=False, asMatch=False): + """The parameters ``pattern`` and ``flags`` are passed + to the ``re.compile()`` function as-is. See the Python + `re module `_ module for an + explanation of the acceptable patterns and flags. + """ super(Regex,self).__init__() if isinstance(pattern, basestring): @@ -2801,7 +3024,7 @@ class Regex(Token): self.pattern = \ self.reString = str(pattern) self.flags = flags - + else: raise ValueError("Regex may only be constructed with a string or a compiled RE object") @@ -2809,6 +3032,8 @@ class Regex(Token): self.errmsg = "Expected " + self.name self.mayIndexError = False self.mayReturnEmpty = True + self.asGroupList = asGroupList + self.asMatch = asMatch def parseImpl( self, instring, loc, doActions=True ): result = self.re.match(instring,loc) @@ -2816,11 +3041,16 @@ class Regex(Token): raise ParseException(instring, loc, self.errmsg, self) loc = result.end() - d = result.groupdict() - ret = ParseResults(result.group()) - if d: - for k in d: - ret[k] = d[k] + if self.asMatch: + ret = result + elif self.asGroupList: + ret = result.groups() + else: + ret = ParseResults(result.group()) + d = result.groupdict() + if d: + for k, v in d.items(): + ret[k] = v return loc,ret def __str__( self ): @@ -2834,28 +3064,70 @@ class Regex(Token): return self.strRepr + def sub(self, repl): + """ + Return Regex with an attached parse action to transform the parsed + result as if called using `re.sub(expr, repl, string) `_. + + Example:: + + make_html = Regex(r"(\w+):(.*?):").sub(r"<\1>\2") + print(make_html.transformString("h1:main title:")) + # prints "

main title

" + """ + if self.asGroupList: + warnings.warn("cannot use sub() with Regex(asGroupList=True)", + SyntaxWarning, stacklevel=2) + raise SyntaxError() + + if self.asMatch and callable(repl): + warnings.warn("cannot use sub() with a callable with Regex(asMatch=True)", + SyntaxWarning, stacklevel=2) + raise SyntaxError() + + if self.asMatch: + def pa(tokens): + return tokens[0].expand(repl) + else: + def pa(tokens): + return self.re.sub(repl, tokens[0]) + return self.addParseAction(pa) class QuotedString(Token): r""" Token for matching strings that are delimited by quoting characters. - + Defined with the following parameters: - - quoteChar - string of one or more characters defining the quote delimiting string - - escChar - character to escape quotes, typically backslash (default=C{None}) - - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None}) - - multiline - boolean indicating whether quotes can span multiple lines (default=C{False}) - - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True}) - - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar) - - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True}) + + - quoteChar - string of one or more characters defining the + quote delimiting string + - escChar - character to escape quotes, typically backslash + (default= ``None`` ) + - escQuote - special quote sequence to escape an embedded quote + string (such as SQL's ``""`` to escape an embedded ``"``) + (default= ``None`` ) + - multiline - boolean indicating whether quotes can span + multiple lines (default= ``False`` ) + - unquoteResults - boolean indicating whether the matched text + should be unquoted (default= ``True`` ) + - endQuoteChar - string of one or more characters defining the + end of the quote delimited string (default= ``None`` => same as + quoteChar) + - convertWhitespaceEscapes - convert escaped whitespace + (``'\t'``, ``'\n'``, etc.) to actual whitespace + (default= ``True`` ) Example:: + qs = QuotedString('"') print(qs.searchString('lsjdf "This is the quote" sldjf')) complex_qs = QuotedString('{{', endQuoteChar='}}') print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf')) sql_qs = QuotedString('"', escQuote='""') print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf')) + prints:: + [['This is the quote']] [['This is the "quote"']] [['This is the quote with "embedded" quotes']] @@ -2973,19 +3245,23 @@ class QuotedString(Token): class CharsNotIn(Token): - """ - Token for matching words composed of characters I{not} in a given set (will - include whitespace in matched characters if not listed in the provided exclusion set - see example). - Defined with string containing all disallowed characters, and an optional - minimum, maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. + """Token for matching words composed of characters *not* in a given + set (will include whitespace in matched characters if not listed in + the provided exclusion set - see example). Defined with string + containing all disallowed characters, and an optional minimum, + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. Example:: + # define a comma-separated-value as anything that is not a ',' csv_value = CharsNotIn(',') print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213")) + prints:: + ['dkls', 'lsdkjf', 's12 34', '@!#', '213'] """ def __init__( self, notChars, min=1, max=0, exact=0 ): @@ -2994,7 +3270,9 @@ class CharsNotIn(Token): self.notChars = notChars if min < 1: - raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted") + raise ValueError( + "cannot specify a minimum length < 1; use " + + "Optional(CharsNotIn()) if zero-length char group is permitted") self.minLen = min @@ -3044,19 +3322,38 @@ class CharsNotIn(Token): return self.strRepr class White(Token): - """ - Special matching class for matching whitespace. Normally, whitespace is ignored - by pyparsing grammars. This class is included when some whitespace structures - are significant. Define with a string containing the whitespace characters to be - matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments, - as defined for the C{L{Word}} class. + """Special matching class for matching whitespace. Normally, + whitespace is ignored by pyparsing grammars. This class is included + when some whitespace structures are significant. Define with + a string containing the whitespace characters to be matched; default + is ``" \\t\\r\\n"``. Also takes optional ``min``, + ``max``, and ``exact`` arguments, as defined for the + :class:`Word` class. """ whiteStrs = { - " " : "", - "\t": "", - "\n": "", - "\r": "", - "\f": "", + ' ' : '', + '\t': '', + '\n': '', + '\r': '', + '\f': '', + 'u\00A0': '', + 'u\1680': '', + 'u\180E': '', + 'u\2000': '', + 'u\2001': '', + 'u\2002': '', + 'u\2003': '', + 'u\2004': '', + 'u\2005': '', + 'u\2006': '', + 'u\2007': '', + 'u\2008': '', + 'u\2009': '', + 'u\200A': '', + 'u\200B': '', + 'u\202F': '', + 'u\205F': '', + 'u\3000': '', } def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0): super(White,self).__init__() @@ -3102,8 +3399,8 @@ class _PositionToken(Token): self.mayIndexError = False class GoToColumn(_PositionToken): - """ - Token to advance to a specific column of input text; useful for tabular report scraping. + """Token to advance to a specific column of input text; useful for + tabular report scraping. """ def __init__( self, colno ): super(GoToColumn,self).__init__() @@ -3128,11 +3425,11 @@ class GoToColumn(_PositionToken): class LineStart(_PositionToken): - """ - Matches if current position is at the beginning of a line within the parse string - + """Matches if current position is at the beginning of a line within + the parse string + Example:: - + test = '''\ AAA this line AAA and this line @@ -3142,10 +3439,11 @@ class LineStart(_PositionToken): for t in (LineStart() + 'AAA' + restOfLine).searchString(test): print(t) - - Prints:: + + prints:: + ['AAA', ' this line'] - ['AAA', ' and this line'] + ['AAA', ' and this line'] """ def __init__( self ): @@ -3158,8 +3456,8 @@ class LineStart(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class LineEnd(_PositionToken): - """ - Matches if current position is at the end of a line within the parse string + """Matches if current position is at the end of a line within the + parse string """ def __init__( self ): super(LineEnd,self).__init__() @@ -3178,8 +3476,8 @@ class LineEnd(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class StringStart(_PositionToken): - """ - Matches if current position is at the beginning of the parse string + """Matches if current position is at the beginning of the parse + string """ def __init__( self ): super(StringStart,self).__init__() @@ -3193,8 +3491,7 @@ class StringStart(_PositionToken): return loc, [] class StringEnd(_PositionToken): - """ - Matches if current position is at the end of the parse string + """Matches if current position is at the end of the parse string """ def __init__( self ): super(StringEnd,self).__init__() @@ -3211,12 +3508,13 @@ class StringEnd(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class WordStart(_PositionToken): - """ - Matches if the current position is at the beginning of a Word, and - is not preceded by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of - the string being parsed, or at the beginning of a line. + """Matches if the current position is at the beginning of a Word, + and is not preceded by any character in a given set of + ``wordChars`` (default= ``printables``). To emulate the + ``\b`` behavior of regular expressions, use + ``WordStart(alphanums)``. ``WordStart`` will also match at + the beginning of the string being parsed, or at the beginning of + a line. """ def __init__(self, wordChars = printables): super(WordStart,self).__init__() @@ -3231,12 +3529,12 @@ class WordStart(_PositionToken): return loc, [] class WordEnd(_PositionToken): - """ - Matches if the current position is at the end of a Word, and - is not followed by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of - the string being parsed, or at the end of a line. + """Matches if the current position is at the end of a Word, and is + not followed by any character in a given set of ``wordChars`` + (default= ``printables``). To emulate the ``\b`` behavior of + regular expressions, use ``WordEnd(alphanums)``. ``WordEnd`` + will also match at the end of the string being parsed, or at the end + of a line. """ def __init__(self, wordChars = printables): super(WordEnd,self).__init__() @@ -3254,8 +3552,8 @@ class WordEnd(_PositionToken): class ParseExpression(ParserElement): - """ - Abstract subclass of ParserElement, for combining and post-processing parsed tokens. + """Abstract subclass of ParserElement, for combining and + post-processing parsed tokens. """ def __init__( self, exprs, savelist = False ): super(ParseExpression,self).__init__(savelist) @@ -3286,7 +3584,7 @@ class ParseExpression(ParserElement): return self def leaveWhitespace( self ): - """Extends C{leaveWhitespace} defined in base class, and also invokes C{leaveWhitespace} on + """Extends ``leaveWhitespace`` defined in base class, and also invokes ``leaveWhitespace`` on all contained expressions.""" self.skipWhitespace = False self.exprs = [ e.copy() for e in self.exprs ] @@ -3347,7 +3645,7 @@ class ParseExpression(ParserElement): self.mayIndexError |= other.mayIndexError self.errmsg = "Expected " + _ustr(self) - + return self def setResultsName( self, name, listAllMatches=False ): @@ -3359,7 +3657,7 @@ class ParseExpression(ParserElement): for e in self.exprs: e.validate(tmp) self.checkRecursion( [] ) - + def copy(self): ret = super(ParseExpression,self).copy() ret.exprs = [e.copy() for e in self.exprs] @@ -3367,12 +3665,14 @@ class ParseExpression(ParserElement): class And(ParseExpression): """ - Requires all given C{ParseExpression}s to be found in the given order. + Requires all given :class:`ParseExpression` s to be found in the given order. Expressions may be separated by whitespace. - May be constructed using the C{'+'} operator. - May also be constructed using the C{'-'} operator, which will suppress backtracking. + May be constructed using the ``'+'`` operator. + May also be constructed using the ``'-'`` operator, which will + suppress backtracking. Example:: + integer = Word(nums) name_expr = OneOrMore(Word(alphas)) @@ -3394,6 +3694,11 @@ class And(ParseExpression): self.skipWhitespace = self.exprs[0].skipWhitespace self.callPreparse = True + def streamline(self): + super(And, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): # pass False as last arg to _parse for first element, since we already # pre-parsed the string as part of our And pre-parsing @@ -3442,17 +3747,20 @@ class And(ParseExpression): class Or(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the expression that matches the longest string will be used. - May be constructed using the C{'^'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the expression that matches the longest + string will be used. May be constructed using the ``'^'`` + operator. Example:: + # construct Or using '^' operator - + number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) + prints:: + [['123'], ['3.1416'], ['789']] """ def __init__( self, exprs, savelist = False ): @@ -3462,6 +3770,11 @@ class Or(ParseExpression): else: self.mayReturnEmpty = True + def streamline(self): + super(Or, self).streamline() + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): maxExcLoc = -1 maxException = None @@ -3521,14 +3834,14 @@ class Or(ParseExpression): class MatchFirst(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the first one listed is the one that will match. - May be constructed using the C{'|'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the first one listed is the one that will + match. May be constructed using the ``'|'`` operator. Example:: + # construct MatchFirst using '|' operator - + # watch the order of expressions to match number = Word(nums) | Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) # Fail! -> [['123'], ['3'], ['1416'], ['789']] @@ -3541,9 +3854,15 @@ class MatchFirst(ParseExpression): super(MatchFirst,self).__init__(exprs, savelist) if self.exprs: self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) + # self.saveAsList = any(e.saveAsList for e in self.exprs) else: self.mayReturnEmpty = True + def streamline(self): + super(MatchFirst, self).streamline() + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): maxExcLoc = -1 maxException = None @@ -3589,12 +3908,13 @@ class MatchFirst(ParseExpression): class Each(ParseExpression): - """ - Requires all given C{ParseExpression}s to be found, but in any order. - Expressions may be separated by whitespace. - May be constructed using the C{'&'} operator. + """Requires all given :class:`ParseExpression` s to be found, but in + any order. Expressions may be separated by whitespace. + + May be constructed using the ``'&'`` operator. Example:: + color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN") shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON") integer = Word(nums) @@ -3603,7 +3923,7 @@ class Each(ParseExpression): color_attr = "color:" + color("color") size_attr = "size:" + integer("size") - # use Each (using operator '&') to accept attributes in any order + # use Each (using operator '&') to accept attributes in any order # (shape and posn are required, color and size are optional) shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr) @@ -3613,7 +3933,9 @@ class Each(ParseExpression): color:GREEN size:20 shape:TRIANGLE posn:20,40 ''' ) + prints:: + shape: SQUARE color: BLACK posn: 100, 120 ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']] - color: BLACK @@ -3647,6 +3969,12 @@ class Each(ParseExpression): self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) self.skipWhitespace = True self.initExprGroups = True + self.saveAsList = True + + def streamline(self): + super(Each, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self def parseImpl( self, instring, loc, doActions=True ): if self.initExprGroups: @@ -3713,8 +4041,8 @@ class Each(ParseExpression): class ParseElementEnhance(ParserElement): - """ - Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens. + """Abstract subclass of :class:`ParserElement`, for combining and + post-processing parsed tokens. """ def __init__( self, expr, savelist=False ): super(ParseElementEnhance,self).__init__(savelist) @@ -3790,20 +4118,25 @@ class ParseElementEnhance(ParserElement): class FollowedBy(ParseElementEnhance): - """ - Lookahead matching of the given parse expression. C{FollowedBy} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression matches at the current - position. C{FollowedBy} always returns a null token list. + """Lookahead matching of the given parse expression. + ``FollowedBy`` does *not* advance the parsing position within + the input string, it only verifies that the specified parse + expression matches at the current position. ``FollowedBy`` + always returns a null token list. If any results names are defined + in the lookahead expression, those *will* be returned for access by + name. Example:: + # use FollowedBy to match a label only if it is followed by a ':' data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint() + prints:: + [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']] """ def __init__( self, expr ): @@ -3811,20 +4144,108 @@ class FollowedBy(ParseElementEnhance): self.mayReturnEmpty = True def parseImpl( self, instring, loc, doActions=True ): - self.expr.tryParse( instring, loc ) - return loc, [] + _, ret = self.expr._parse(instring, loc, doActions=doActions) + del ret[:] + return loc, ret + + +class PrecededBy(ParseElementEnhance): + """Lookbehind matching of the given parse expression. + ``PrecededBy`` does not advance the parsing position within the + input string, it only verifies that the specified parse expression + matches prior to the current position. ``PrecededBy`` always + returns a null token list, but if a results name is defined on the + given expression, it is returned. + + Parameters: + + - expr - expression that must match prior to the current parse + location + - retreat - (default= ``None``) - (int) maximum number of characters + to lookbehind prior to the current parse location + + If the lookbehind expression is a string, Literal, Keyword, or + a Word or CharsNotIn with a specified exact or maximum length, then + the retreat parameter is not required. Otherwise, retreat must be + specified to give a maximum number of characters to look back from + the current parse position for a lookbehind match. + + Example:: + + # VB-style variable names with type prefixes + int_var = PrecededBy("#") + pyparsing_common.identifier + str_var = PrecededBy("$") + pyparsing_common.identifier + + """ + def __init__(self, expr, retreat=None): + super(PrecededBy, self).__init__(expr) + self.expr = self.expr().leaveWhitespace() + self.mayReturnEmpty = True + self.mayIndexError = False + self.exact = False + if isinstance(expr, str): + retreat = len(expr) + self.exact = True + elif isinstance(expr, (Literal, Keyword)): + retreat = expr.matchLen + self.exact = True + elif isinstance(expr, (Word, CharsNotIn)) and expr.maxLen != _MAX_INT: + retreat = expr.maxLen + self.exact = True + elif isinstance(expr, _PositionToken): + retreat = 0 + self.exact = True + self.retreat = retreat + self.errmsg = "not preceded by " + str(expr) + self.skipWhitespace = False + + def parseImpl(self, instring, loc=0, doActions=True): + if self.exact: + if loc < self.retreat: + raise ParseException(instring, loc, self.errmsg) + start = loc - self.retreat + _, ret = self.expr._parse(instring, start) + else: + # retreat specified a maximum lookbehind window, iterate + test_expr = self.expr + StringEnd() + instring_slice = instring[:loc] + last_expr = ParseException(instring, loc, self.errmsg) + for offset in range(1, min(loc, self.retreat+1)): + try: + _, ret = test_expr._parse(instring_slice, loc-offset) + except ParseBaseException as pbe: + last_expr = pbe + else: + break + else: + raise last_expr + # return empty list of tokens, but preserve any defined results names + del ret[:] + return loc, ret class NotAny(ParseElementEnhance): - """ - Lookahead to disallow matching with the given parse expression. C{NotAny} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression does I{not} match at the current - position. Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny} - always returns a null token list. May be constructed using the '~' operator. + """Lookahead to disallow matching with the given parse expression. + ``NotAny`` does *not* advance the parsing position within the + input string, it only verifies that the specified parse expression + does *not* match at the current position. Also, ``NotAny`` does + *not* skip over leading whitespace. ``NotAny`` always returns + a null token list. May be constructed using the '~' operator. Example:: - + + AND, OR, NOT = map(CaselessKeyword, "AND OR NOT".split()) + + # take care not to mistake keywords for identifiers + ident = ~(AND | OR | NOT) + Word(alphas) + boolean_term = Optional(NOT) + ident + + # very crude boolean expression - to support parenthesis groups and + # operation hierarchy, use infixNotation + boolean_expr = boolean_term + ZeroOrMore((AND | OR) + boolean_term) + + # integers that are followed by "." are actually floats + integer = Word(nums) + ~Char(".") """ def __init__( self, expr ): super(NotAny,self).__init__(expr) @@ -3862,7 +4283,7 @@ class _MultipleMatch(ParseElementEnhance): check_ender = self.not_ender is not None if check_ender: try_not_ender = self.not_ender.tryParse - + # must be at least one (but first see if we are the stopOn sentinel; # if so, fail) if check_ender: @@ -3884,18 +4305,18 @@ class _MultipleMatch(ParseElementEnhance): pass return loc, tokens - + class OneOrMore(_MultipleMatch): - """ - Repetition of one or more of the given expression. - + """Repetition of one or more of the given expression. + Parameters: - expr - expression that must match one or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) @@ -3906,7 +4327,7 @@ class OneOrMore(_MultipleMatch): # use stopOn attribute for OneOrMore to avoid reading label string as part of the data attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']] - + # could also be written as (attr_expr * (1,)).parseString(text).pprint() """ @@ -3921,21 +4342,20 @@ class OneOrMore(_MultipleMatch): return self.strRepr class ZeroOrMore(_MultipleMatch): - """ - Optional repetition of zero or more of the given expression. - + """Optional repetition of zero or more of the given expression. + Parameters: - expr - expression that must match zero or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) - Example: similar to L{OneOrMore} + Example: similar to :class:`OneOrMore` """ def __init__( self, expr, stopOn=None): super(ZeroOrMore,self).__init__(expr, stopOn=stopOn) self.mayReturnEmpty = True - + def parseImpl( self, instring, loc, doActions=True ): try: return super(ZeroOrMore, self).parseImpl(instring, loc, doActions) @@ -3960,27 +4380,29 @@ class _NullToken(object): _optionalNotMatched = _NullToken() class Optional(ParseElementEnhance): - """ - Optional matching of the given expression. + """Optional matching of the given expression. Parameters: - expr - expression that must match zero or more times - default (optional) - value to be returned if the optional expression is not found. Example:: + # US postal code can be a 5-digit zip, plus optional 4-digit qualifier zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4))) zip.runTests(''' # traditional ZIP code 12345 - + # ZIP+4 form 12101-0001 - + # invalid ZIP 98765- ''') + prints:: + # traditional ZIP code 12345 ['12345'] @@ -4024,20 +4446,21 @@ class Optional(ParseElementEnhance): return self.strRepr class SkipTo(ParseElementEnhance): - """ - Token for skipping over all undefined text until the matched expression is found. + """Token for skipping over all undefined text until the matched + expression is found. Parameters: - expr - target expression marking the end of the data to be skipped - - include - (default=C{False}) if True, the target expression is also parsed + - include - (default= ``False``) if True, the target expression is also parsed (the skipped text and target expression are returned as a 2-element list). - - ignore - (default=C{None}) used to define grammars (typically quoted strings and + - ignore - (default= ``None``) used to define grammars (typically quoted strings and comments) that might contain false matches to the target expression - - failOn - (default=C{None}) define expressions that are not allowed to be - included in the skipped test; if found before the target expression is found, + - failOn - (default= ``None``) define expressions that are not allowed to be + included in the skipped test; if found before the target expression is found, the SkipTo is not a match Example:: + report = ''' Outstanding Issues Report - 1 Jan 2000 @@ -4054,14 +4477,16 @@ class SkipTo(ParseElementEnhance): # - parse action will call token.strip() for each matched token, i.e., the description body string_data = SkipTo(SEP, ignore=quotedString) string_data.setParseAction(tokenMap(str.strip)) - ticket_expr = (integer("issue_num") + SEP - + string_data("sev") + SEP - + string_data("desc") + SEP + ticket_expr = (integer("issue_num") + SEP + + string_data("sev") + SEP + + string_data("desc") + SEP + integer("days_open")) - + for tkt in ticket_expr.searchString(report): print tkt.dump() + prints:: + ['101', 'Critical', 'Intermittent system crash', '6'] - days_open: 6 - desc: Intermittent system crash @@ -4084,7 +4509,7 @@ class SkipTo(ParseElementEnhance): self.mayReturnEmpty = True self.mayIndexError = False self.includeMatch = include - self.asList = False + self.saveAsList = False if isinstance(failOn, basestring): self.failOn = ParserElement._literalStringClass(failOn) else: @@ -4098,14 +4523,14 @@ class SkipTo(ParseElementEnhance): expr_parse = self.expr._parse self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None - + tmploc = loc while tmploc <= instrlen: if self_failOn_canParseNext is not None: # break if failOn expression matches if self_failOn_canParseNext(instring, tmploc): break - + if self_ignoreExpr_tryParse is not None: # advance past ignore expressions while 1: @@ -4113,7 +4538,7 @@ class SkipTo(ParseElementEnhance): tmploc = self_ignoreExpr_tryParse(instring, tmploc) except ParseBaseException: break - + try: expr_parse(instring, tmploc, doActions=False, callPreParse=False) except (ParseException, IndexError): @@ -4131,7 +4556,7 @@ class SkipTo(ParseElementEnhance): loc = tmploc skiptext = instring[startloc:loc] skipresult = ParseResults(skiptext) - + if self.includeMatch: loc, mat = expr_parse(instring,loc,doActions,callPreParse=False) skipresult += mat @@ -4139,23 +4564,31 @@ class SkipTo(ParseElementEnhance): return loc, skipresult class Forward(ParseElementEnhance): - """ - Forward declaration of an expression to be defined later - + """Forward declaration of an expression to be defined later - used for recursive grammars, such as algebraic infix notation. - When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator. + When the expression is known, it is assigned to the ``Forward`` + variable using the '<<' operator. + + Note: take care when assigning to ``Forward`` not to overlook + precedence of operators. - Note: take care when assigning to C{Forward} not to overlook precedence of operators. Specifically, '|' has a lower precedence than '<<', so that:: + fwdExpr << a | b | c + will actually be evaluated as:: + (fwdExpr << a) | b | c + thereby leaving b and c out as parseable alternatives. It is recommended that you - explicitly group the values inserted into the C{Forward}:: + explicitly group the values inserted into the ``Forward``:: + fwdExpr << (a | b | c) + Converting to use the '<<=' operator instead will avoid this problem. - See L{ParseResults.pprint} for an example of a recursive parser created using - C{Forward}. + See :class:`ParseResults.pprint` for an example of a recursive + parser created using ``Forward``. """ def __init__( self, other=None ): super(Forward,self).__init__( other, savelist=False ) @@ -4172,10 +4605,10 @@ class Forward(ParseElementEnhance): self.saveAsList = self.expr.saveAsList self.ignoreExprs.extend(self.expr.ignoreExprs) return self - + def __ilshift__(self, other): return self << other - + def leaveWhitespace( self ): self.skipWhitespace = False return self @@ -4225,19 +4658,20 @@ class _ForwardNoRecurse(Forward): class TokenConverter(ParseElementEnhance): """ - Abstract subclass of C{ParseExpression}, for converting parsed results. + Abstract subclass of :class:`ParseExpression`, for converting parsed results. """ def __init__( self, expr, savelist=False ): super(TokenConverter,self).__init__( expr )#, savelist ) self.saveAsList = False class Combine(TokenConverter): - """ - Converter to concatenate all matching tokens to a single string. - By default, the matching patterns must also be contiguous in the input string; - this can be disabled by specifying C{'adjacent=False'} in the constructor. + """Converter to concatenate all matching tokens to a single string. + By default, the matching patterns must also be contiguous in the + input string; this can be disabled by specifying + ``'adjacent=False'`` in the constructor. Example:: + real = Word(nums) + '.' + Word(nums) print(real.parseString('3.1416')) # -> ['3', '.', '1416'] # will also erroneously match the following @@ -4276,10 +4710,11 @@ class Combine(TokenConverter): return retToks class Group(TokenConverter): - """ - Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions. + """Converter to return the matched tokens as a list - useful for + returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions. Example:: + ident = Word(alphas) num = Word(nums) term = ident | num @@ -4291,38 +4726,40 @@ class Group(TokenConverter): """ def __init__( self, expr ): super(Group,self).__init__( expr ) - self.saveAsList = True + self.saveAsList = expr.saveAsList def postParse( self, instring, loc, tokenlist ): return [ tokenlist ] class Dict(TokenConverter): - """ - Converter to return a repetitive expression as a list, but also as a dictionary. - Each element can also be referenced using the first token in the expression as its key. - Useful for tabular report scraping when the first column can be used as a item key. + """Converter to return a repetitive expression as a list, but also + as a dictionary. Each element can also be referenced using the first + token in the expression as its key. Useful for tabular report + scraping when the first column can be used as a item key. Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + # print attributes as plain groups print(OneOrMore(attr_expr).parseString(text).dump()) - + # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names result = Dict(OneOrMore(Group(attr_expr))).parseString(text) print(result.dump()) - - # access named fields as dict entries, or output as dict - print(result['shape']) - print(result.asDict()) - prints:: - ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] + # access named fields as dict entries, or output as dict + print(result['shape']) + print(result.asDict()) + + prints:: + + ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4330,7 +4767,8 @@ class Dict(TokenConverter): - texture: burlap SQUARE {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'} - See more examples at L{ParseResults} of accessing fields by results name. + + See more examples at :class:`ParseResults` of accessing fields by results name. """ def __init__( self, expr ): super(Dict,self).__init__( expr ) @@ -4362,10 +4800,10 @@ class Dict(TokenConverter): class Suppress(TokenConverter): - """ - Converter for ignoring the results of a parsed expression. + """Converter for ignoring the results of a parsed expression. Example:: + source = "a, b, c,d" wd = Word(alphas) wd_list1 = wd + ZeroOrMore(',' + wd) @@ -4375,10 +4813,13 @@ class Suppress(TokenConverter): # way afterward - use Suppress to keep them out of the parsed output wd_list2 = wd + ZeroOrMore(Suppress(',') + wd) print(wd_list2.parseString(source)) + prints:: + ['a', ',', 'b', ',', 'c', ',', 'd'] ['a', 'b', 'c', 'd'] - (See also L{delimitedList}.) + + (See also :class:`delimitedList`.) """ def postParse( self, instring, loc, tokenlist ): return [] @@ -4388,8 +4829,7 @@ class Suppress(TokenConverter): class OnlyOnce(object): - """ - Wrapper for parse actions, to ensure they are only called once. + """Wrapper for parse actions, to ensure they are only called once. """ def __init__(self, methodCall): self.callable = _trim_arity(methodCall) @@ -4404,13 +4844,15 @@ class OnlyOnce(object): self.called = False def traceParseAction(f): - """ - Decorator for debugging parse actions. - - When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".} - When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised. + """Decorator for debugging parse actions. + + When the parse action is called, this decorator will print + ``">> entering method-name(line:, , )"``. + When the parse action completes, the decorator will print + ``"<<"`` followed by the returned value, or any exception that the parse action raised. Example:: + wd = Word(alphas) @traceParseAction @@ -4419,7 +4861,9 @@ def traceParseAction(f): wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) print(wds.parseString("slkdjs sld sldd sdlf sdljf")) + prints:: + >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {})) < ['aa', 'bb', 'cc'] delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE'] """ @@ -4467,16 +4913,21 @@ def delimitedList( expr, delim=",", combine=False ): return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName) def countedArray( expr, intExpr=None ): - """ - Helper to define a counted list of expressions. + """Helper to define a counted list of expressions. + This helper defines a pattern of the form:: + integer expr expr expr... + where the leading integer tells how many expr expressions follow. - The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed. - - If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value. + The matched tokens returns the array of expr tokens as a list - the + leading count token is suppressed. + + If ``intExpr`` is specified, it should be a pyparsing expression + that produces an integer value. Example:: + countedArray(Word(alphas)).parseString('2 ab cd ef') # -> ['ab', 'cd'] # in this parser, the leading integer value is given in binary, @@ -4507,17 +4958,19 @@ def _flatten(L): return ret def matchPreviousLiteral(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousLiteral(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches a - previous literal, will also match the leading C{"1:1"} in C{"1:10"}. - If this is not desired, use C{matchPreviousExpr}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches a previous literal, will also match the leading + ``"1:1"`` in ``"1:10"``. If this is not desired, use + :class:`matchPreviousExpr`. Do *not* use with packrat parsing + enabled. """ rep = Forward() def copyTokenToRepeater(s,l,t): @@ -4535,18 +4988,19 @@ def matchPreviousLiteral(expr): return rep def matchPreviousExpr(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousExpr(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches by - expressions, will I{not} match the leading C{"1:1"} in C{"1:10"}; - the expressions are evaluated first, and then compared, so - C{"1"} is compared with C{"10"}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches by expressions, will *not* match the leading ``"1:1"`` + in ``"1:10"``; the expressions are evaluated first, and then + compared, so ``"1"`` is compared with ``"10"``. Do *not* use + with packrat parsing enabled. """ rep = Forward() e2 = expr.copy() @@ -4571,26 +5025,33 @@ def _escapeRegexRangeChars(s): return _ustr(s) def oneOf( strs, caseless=False, useRegex=True ): - """ - Helper to quickly define a set of alternative Literals, and makes sure to do - longest-first testing when there is a conflict, regardless of the input order, - but returns a C{L{MatchFirst}} for best performance. + """Helper to quickly define a set of alternative Literals, and makes + sure to do longest-first testing when there is a conflict, + regardless of the input order, but returns + a :class:`MatchFirst` for best performance. Parameters: - - strs - a string of space-delimited literals, or a collection of string literals - - caseless - (default=C{False}) - treat all literals as caseless - - useRegex - (default=C{True}) - as an optimization, will generate a Regex - object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or - if creating a C{Regex} raises an exception) + + - strs - a string of space-delimited literals, or a collection of + string literals + - caseless - (default= ``False``) - treat all literals as + caseless + - useRegex - (default= ``True``) - as an optimization, will + generate a Regex object; otherwise, will generate + a :class:`MatchFirst` object (if ``caseless=True``, or if + creating a :class:`Regex` raises an exception) Example:: + comp_oper = oneOf("< = > <= >= !=") var = Word(alphas) number = Word(nums) term = var | number comparison_expr = term + comp_oper + term print(comparison_expr.searchString("B = 12 AA=23 B<=AA AA>12")) + prints:: + [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']] """ if caseless: @@ -4644,19 +5105,21 @@ def oneOf( strs, caseless=False, useRegex=True ): return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols)) def dictOf( key, value ): - """ - Helper to easily and clearly define a dictionary by specifying the respective patterns - for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens - in the proper order. The key pattern can include delimiting markers or punctuation, - as long as they are suppressed, thereby leaving the significant key text. The value - pattern can include named results, so that the C{Dict} results can include named token - fields. + """Helper to easily and clearly define a dictionary by specifying + the respective patterns for the key and value. Takes care of + defining the :class:`Dict`, :class:`ZeroOrMore`, and + :class:`Group` tokens in the proper order. The key pattern + can include delimiting markers or punctuation, as long as they are + suppressed, thereby leaving the significant key text. The value + pattern can include named results, so that the :class:`Dict` results + can include named token fields. Example:: + text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) print(OneOrMore(attr_expr).parseString(text).dump()) - + attr_label = label attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join) @@ -4666,7 +5129,9 @@ def dictOf( key, value ): print(result['shape']) print(result.shape) # object attribute access works too print(result.asDict()) + prints:: + [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4676,29 +5141,34 @@ def dictOf( key, value ): SQUARE {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'} """ - return Dict( ZeroOrMore( Group ( key + value ) ) ) + return Dict(OneOrMore(Group(key + value))) def originalTextFor(expr, asString=True): - """ - Helper to return the original, untokenized text for a given expression. Useful to - restore the parsed fields of an HTML start tag into the raw tag text itself, or to - revert separate tokens with intervening whitespace back to the original matching - input text. By default, returns astring containing the original parsed text. - - If the optional C{asString} argument is passed as C{False}, then the return value is a - C{L{ParseResults}} containing any results names that were originally matched, and a - single token containing the original matched text from the input string. So if - the expression passed to C{L{originalTextFor}} contains expressions with defined - results names, you must set C{asString} to C{False} if you want to preserve those - results name values. + """Helper to return the original, untokenized text for a given + expression. Useful to restore the parsed fields of an HTML start + tag into the raw tag text itself, or to revert separate tokens with + intervening whitespace back to the original matching input text. By + default, returns astring containing the original parsed text. + + If the optional ``asString`` argument is passed as + ``False``, then the return value is + a :class:`ParseResults` containing any results names that + were originally matched, and a single token containing the original + matched text from the input string. So if the expression passed to + :class:`originalTextFor` contains expressions with defined + results names, you must set ``asString`` to ``False`` if you + want to preserve those results name values. Example:: + src = "this is test bold text normal text " for tag in ("b","i"): opener,closer = makeHTMLTags(tag) patt = originalTextFor(opener + SkipTo(closer) + closer) print(patt.searchString(src)[0]) + prints:: + [' bold text '] ['text'] """ @@ -4715,29 +5185,33 @@ def originalTextFor(expr, asString=True): matchExpr.ignoreExprs = expr.ignoreExprs return matchExpr -def ungroup(expr): - """ - Helper to undo pyparsing's default grouping of And expressions, even - if all but one are non-empty. +def ungroup(expr): + """Helper to undo pyparsing's default grouping of And expressions, + even if all but one are non-empty. """ return TokenConverter(expr).setParseAction(lambda t:t[0]) def locatedExpr(expr): - """ - Helper to decorate a returned token with its starting and ending locations in the input string. + """Helper to decorate a returned token with its starting and ending + locations in the input string. + This helper adds the following results names: + - locn_start = location where matched expression begins - locn_end = location where matched expression ends - value = the actual parsed results - Be careful if the input text contains C{} characters, you may want to call - C{L{ParserElement.parseWithTabs}} + Be careful if the input text contains ```` characters, you + may want to call :class:`ParserElement.parseWithTabs` Example:: + wd = Word(alphas) for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"): print(match) + prints:: + [[0, 'ljsdf', 5]] [[8, 'lksdjjf', 15]] [[18, 'lkkjj', 23]] @@ -4761,22 +5235,30 @@ _charRange = Group(_singleChar + Suppress("-") + _singleChar) _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" def srange(s): - r""" - Helper to easily define string ranges for use in Word construction. Borrows - syntax from regexp '[]' string range definitions:: + r"""Helper to easily define string ranges for use in Word + construction. Borrows syntax from regexp '[]' string range + definitions:: + srange("[0-9]") -> "0123456789" srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz" srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_" - The input string must be enclosed in []'s, and the returned string is the expanded - character set joined into a single string. - The values enclosed in the []'s may be: + + The input string must be enclosed in []'s, and the returned string + is the expanded character set joined into a single string. The + values enclosed in the []'s may be: + - a single character - - an escaped character with a leading backslash (such as C{\-} or C{\]}) - - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character) - (C{\0x##} is also supported for backwards compatibility) - - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character) - - a range of any of the above, separated by a dash (C{'a-z'}, etc.) - - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.) + - an escaped character with a leading backslash (such as ``\-`` + or ``\]``) + - an escaped hex character with a leading ``'\x'`` + (``\x21``, which is a ``'!'`` character) (``\0x##`` + is also supported for backwards compatibility) + - an escaped octal character with a leading ``'\0'`` + (``\041``, which is a ``'!'`` character) + - a range of any of the above, separated by a dash (``'a-z'``, + etc.) + - any combination of the above (``'aeiouy'``, + ``'a-zA-Z0-9_$'``, etc.) """ _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1)) try: @@ -4785,9 +5267,8 @@ def srange(s): return "" def matchOnlyAtCol(n): - """ - Helper method for defining parse actions that require matching at a specific - column in the input text. + """Helper method for defining parse actions that require matching at + a specific column in the input text. """ def verifyCol(strg,locn,toks): if col(locn,strg) != n: @@ -4795,24 +5276,26 @@ def matchOnlyAtCol(n): return verifyCol def replaceWith(replStr): - """ - Helper method for common parse actions that simply return a literal value. Especially - useful when used with C{L{transformString}()}. + """Helper method for common parse actions that simply return + a literal value. Especially useful when used with + :class:`transformString` (). Example:: + num = Word(nums).setParseAction(lambda toks: int(toks[0])) na = oneOf("N/A NA").setParseAction(replaceWith(math.nan)) term = na | num - + OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234] """ return lambda s,l,t: [replStr] def removeQuotes(s,l,t): - """ - Helper parse action for removing quotation marks from parsed quoted strings. + """Helper parse action for removing quotation marks from parsed + quoted strings. Example:: + # by default, quotation marks are included in parsed results quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"] @@ -4823,18 +5306,20 @@ def removeQuotes(s,l,t): return t[0][1:-1] def tokenMap(func, *args): - """ - Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional - args are passed, they are forwarded to the given function as additional arguments after - the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the - parsed data to an integer using base 16. + """Helper to define a parse action by mapping a function to all + elements of a ParseResults list. If any additional args are passed, + they are forwarded to the given function as additional arguments + after the token, as in + ``hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))``, + which will convert the parsed data to an integer using base 16. + + Example (compare the last to example in :class:`ParserElement.transformString`:: - Example (compare the last to example in L{ParserElement.transformString}:: hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16)) hex_ints.runTests(''' 00 11 22 aa FF 0a 0d 1a ''') - + upperword = Word(alphas).setParseAction(tokenMap(str.upper)) OneOrMore(upperword).runTests(''' my kingdom for a horse @@ -4844,7 +5329,9 @@ def tokenMap(func, *args): OneOrMore(wd).setParseAction(' '.join).runTests(''' now is the winter of our discontent made glorious summer by this sun of york ''') + prints:: + 00 11 22 aa FF 0a 0d 1a [0, 17, 34, 170, 255, 10, 13, 26] @@ -4858,7 +5345,7 @@ def tokenMap(func, *args): return [func(tokn, *args) for tokn in t] try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -4867,11 +5354,13 @@ def tokenMap(func, *args): return pa upcaseTokens = tokenMap(lambda t: _ustr(t).upper()) -"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}""" +"""(Deprecated) Helper parse action to convert tokens to upper case. +Deprecated in favor of :class:`pyparsing_common.upcaseTokens`""" downcaseTokens = tokenMap(lambda t: _ustr(t).lower()) -"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}""" - +"""(Deprecated) Helper parse action to convert tokens to lower case. +Deprecated in favor of :class:`pyparsing_common.downcaseTokens`""" + def _makeTags(tagStr, xml): """Internal helper to construct opening and closing tag expressions, given a tag name""" if isinstance(tagStr,basestring): @@ -4902,55 +5391,63 @@ def _makeTags(tagStr, xml): return openTag, closeTag def makeHTMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches - tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values. + """Helper to construct opening and closing tag expressions for HTML, + given a tag name. Matches tags in either upper or lower case, + attributes with namespaces and with quoted or unquoted values. Example:: - text = 'More info at the pyparsing wiki page' - # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple + + text = 'More info at the pyparsing wiki page' + # makeHTMLTags returns pyparsing expressions for the opening and + # closing tags as a 2-tuple a,a_end = makeHTMLTags("A") link_expr = a + SkipTo(a_end)("link_text") + a_end - + for link in link_expr.searchString(text): - # attributes in the tag (like "href" shown here) are also accessible as named results + # attributes in the tag (like "href" shown here) are + # also accessible as named results print(link.link_text, '->', link.href) + prints:: - pyparsing -> http://pyparsing.wikispaces.com + + pyparsing -> https://github.com/pyparsing/pyparsing/wiki """ return _makeTags( tagStr, False ) def makeXMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for XML, given a tag name. Matches - tags only in the given upper/lower case. + """Helper to construct opening and closing tag expressions for XML, + given a tag name. Matches tags only in the given upper/lower case. - Example: similar to L{makeHTMLTags} + Example: similar to :class:`makeHTMLTags` """ return _makeTags( tagStr, True ) def withAttribute(*args,**attrDict): - """ - Helper to create a validating parse action to be used with start tags created - with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag - with a required attribute value, to avoid false matches on common tags such as - C{} or C{
}. + """Helper to create a validating parse action to be used with start + tags created with :class:`makeXMLTags` or + :class:`makeHTMLTags`. Use ``withAttribute`` to qualify + a starting tag with a required attribute value, to avoid false + matches on common tags such as ```` or ``
``. - Call C{withAttribute} with a series of attribute names and values. Specify the list - of filter attributes names and values as: - - keyword arguments, as in C{(align="right")}, or - - as an explicit dict with C{**} operator, when an attribute name is also a Python - reserved word, as in C{**{"class":"Customer", "align":"right"}} - - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") ) - For attribute names with a namespace prefix, you must use the second form. Attribute - names are matched insensitive to upper/lower case. - - If just testing for C{class} (with or without a namespace), use C{L{withClass}}. + Call ``withAttribute`` with a series of attribute names and + values. Specify the list of filter attributes names and values as: - To verify that the attribute exists, but without specifying a value, pass - C{withAttribute.ANY_VALUE} as the value. + - keyword arguments, as in ``(align="right")``, or + - as an explicit dict with ``**`` operator, when an attribute + name is also a Python reserved word, as in ``**{"class":"Customer", "align":"right"}`` + - a list of name-value tuples, as in ``(("ns1:class", "Customer"), ("ns2:align","right"))`` + + For attribute names with a namespace prefix, you must use the second + form. Attribute names are matched insensitive to upper/lower case. + + If just testing for ``class`` (with or without a namespace), use + :class:`withClass`. + + To verify that the attribute exists, but without specifying a value, + pass ``withAttribute.ANY_VALUE`` as the value. Example:: + html = '''
Some text @@ -4958,7 +5455,7 @@ def withAttribute(*args,**attrDict):
1,3 2,3 1,1
this has no type
- + ''' div,div_end = makeHTMLTags("div") @@ -4967,13 +5464,15 @@ def withAttribute(*args,**attrDict): grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + # construct a match with any div tag having a type attribute, regardless of the value div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 @@ -4995,11 +5494,12 @@ def withAttribute(*args,**attrDict): withAttribute.ANY_VALUE = object() def withClass(classname, namespace=''): - """ - Simplified version of C{L{withAttribute}} when matching on a div class - made - difficult because C{class} is a reserved word in Python. + """Simplified version of :class:`withAttribute` when + matching on a div class - made difficult because ``class`` is + a reserved word in Python. Example:: + html = '''
Some text @@ -5007,84 +5507,96 @@ def withClass(classname, namespace=''):
1,3 2,3 1,1
this <div> has no class
- + ''' div,div_end = makeHTMLTags("div") div_grid = div().setParseAction(withClass("grid")) - + grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 1,3 2,3 1,1 """ classattr = "%s:class" % namespace if namespace else "class" - return withAttribute(**{classattr : classname}) + return withAttribute(**{classattr : classname}) -opAssoc = _Constants() +opAssoc = SimpleNamespace() opAssoc.LEFT = object() opAssoc.RIGHT = object() def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): - """ - Helper method for constructing grammars of expressions made up of - operators working in a precedence hierarchy. Operators may be unary or - binary, left- or right-associative. Parse actions can also be attached - to operator expressions. The generated parser will also recognize the use - of parentheses to override operator precedences (see example below). - - Note: if you define a deep operator list, you may see performance issues - when using infixNotation. See L{ParserElement.enablePackrat} for a - mechanism to potentially improve your parser performance. + """Helper method for constructing grammars of expressions made up of + operators working in a precedence hierarchy. Operators may be unary + or binary, left- or right-associative. Parse actions can also be + attached to operator expressions. The generated parser will also + recognize the use of parentheses to override operator precedences + (see example below). + + Note: if you define a deep operator list, you may see performance + issues when using infixNotation. See + :class:`ParserElement.enablePackrat` for a mechanism to potentially + improve your parser performance. Parameters: - - baseExpr - expression representing the most basic element for the nested - - opList - list of tuples, one for each operator precedence level in the - expression grammar; each tuple is of the form - (opExpr, numTerms, rightLeftAssoc, parseAction), where: - - opExpr is the pyparsing expression for the operator; - may also be a string, which will be converted to a Literal; - if numTerms is 3, opExpr is a tuple of two expressions, for the - two operators separating the 3 terms - - numTerms is the number of terms for this operator (must - be 1, 2, or 3) - - rightLeftAssoc is the indicator whether the operator is - right or left associative, using the pyparsing-defined - constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}. + - baseExpr - expression representing the most basic element for the + nested + - opList - list of tuples, one for each operator precedence level + in the expression grammar; each tuple is of the form ``(opExpr, + numTerms, rightLeftAssoc, parseAction)``, where: + + - opExpr is the pyparsing expression for the operator; may also + be a string, which will be converted to a Literal; if numTerms + is 3, opExpr is a tuple of two expressions, for the two + operators separating the 3 terms + - numTerms is the number of terms for this operator (must be 1, + 2, or 3) + - rightLeftAssoc is the indicator whether the operator is right + or left associative, using the pyparsing-defined constants + ``opAssoc.RIGHT`` and ``opAssoc.LEFT``. - parseAction is the parse action to be associated with - expressions matching this operator expression (the - parse action tuple member may be omitted); if the parse action - is passed a tuple or list of functions, this is equivalent to - calling C{setParseAction(*fn)} (L{ParserElement.setParseAction}) - - lpar - expression for matching left-parentheses (default=C{Suppress('(')}) - - rpar - expression for matching right-parentheses (default=C{Suppress(')')}) + expressions matching this operator expression (the parse action + tuple member may be omitted); if the parse action is passed + a tuple or list of functions, this is equivalent to calling + ``setParseAction(*fn)`` + (:class:`ParserElement.setParseAction`) + - lpar - expression for matching left-parentheses + (default= ``Suppress('(')``) + - rpar - expression for matching right-parentheses + (default= ``Suppress(')')``) Example:: - # simple example of four-function arithmetic with ints and variable names + + # simple example of four-function arithmetic with ints and + # variable names integer = pyparsing_common.signed_integer - varname = pyparsing_common.identifier - + varname = pyparsing_common.identifier + arith_expr = infixNotation(integer | varname, [ ('-', 1, opAssoc.RIGHT), (oneOf('* /'), 2, opAssoc.LEFT), (oneOf('+ -'), 2, opAssoc.LEFT), ]) - + arith_expr.runTests(''' 5+3*6 (5+3)*6 -2--11 ''', fullDump=False) + prints:: + 5+3*6 [[5, '+', [3, '*', 6]]] @@ -5094,6 +5606,12 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): -2--11 [[['-', 2], '-', ['-', 11]]] """ + # captive version of FollowedBy that does not do parse actions or capture results names + class _FB(FollowedBy): + def parseImpl(self, instring, loc, doActions=True): + self.expr.tryParse(instring, loc) + return loc, [] + ret = Forward() lastExpr = baseExpr | ( lpar + ret + rpar ) for i,operDef in enumerate(opList): @@ -5101,19 +5619,20 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr if arity == 3: if opExpr is None or len(opExpr) != 2: - raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions") + raise ValueError( + "if numterms=3, opExpr must be a tuple or list of two expressions") opExpr1, opExpr2 = opExpr thisExpr = Forward().setName(termName) if rightLeftAssoc == opAssoc.LEFT: if arity == 1: - matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) + matchExpr = _FB(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) + matchExpr = _FB(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) else: - matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) + matchExpr = _FB(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ + matchExpr = _FB(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr ) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") @@ -5122,14 +5641,14 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): # try to avoid LR with this extra test if not isinstance(opExpr, Optional): opExpr = Optional(opExpr) - matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) + matchExpr = _FB(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) + matchExpr = _FB(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) else: - matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) + matchExpr = _FB(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ + matchExpr = _FB(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr ) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") @@ -5146,7 +5665,8 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): return ret operatorPrecedence = infixNotation -"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release.""" +"""(Deprecated) Former name of :class:`infixNotation`, will be +dropped in a future release.""" dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes") sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes") @@ -5155,28 +5675,33 @@ quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+) unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal") def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()): - """ - Helper method for defining nested lists enclosed in opening and closing - delimiters ("(" and ")" are the default). + """Helper method for defining nested lists enclosed in opening and + closing delimiters ("(" and ")" are the default). Parameters: - - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression - - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression - - content - expression for items within the nested lists (default=C{None}) - - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString}) + - opener - opening character for a nested list + (default= ``"("``); can also be a pyparsing expression + - closer - closing character for a nested list + (default= ``")"``); can also be a pyparsing expression + - content - expression for items within the nested lists + (default= ``None``) + - ignoreExpr - expression for ignoring opening and closing + delimiters (default= :class:`quotedString`) - If an expression is not provided for the content argument, the nested - expression will capture all whitespace-delimited content between delimiters - as a list of separate values. + If an expression is not provided for the content argument, the + nested expression will capture all whitespace-delimited content + between delimiters as a list of separate values. - Use the C{ignoreExpr} argument to define expressions that may contain - opening or closing characters that should not be treated as opening - or closing characters for nesting, such as quotedString or a comment - expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}. - The default is L{quotedString}, but if no expressions are to be ignored, - then pass C{None} for this argument. + Use the ``ignoreExpr`` argument to define expressions that may + contain opening or closing characters that should not be treated as + opening or closing characters for nesting, such as quotedString or + a comment expression. Specify multiple expressions using an + :class:`Or` or :class:`MatchFirst`. The default is + :class:`quotedString`, but if no expressions are to be ignored, then + pass ``None`` for this argument. Example:: + data_type = oneOf("void int short long char float double") decl_data_type = Combine(data_type + Optional(Word('*'))) ident = Word(alphas+'_', alphanums+'_') @@ -5186,29 +5711,31 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment)) - c_function = (decl_data_type("type") + c_function = (decl_data_type("type") + ident("name") - + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + code_body("body")) c_function.ignore(cStyleComment) - + source_code = ''' - int is_odd(int x) { - return (x%2); + int is_odd(int x) { + return (x%2); } - - int dec_to_hex(char hchar) { - if (hchar >= '0' && hchar <= '9') { - return (ord(hchar)-ord('0')); - } else { + + int dec_to_hex(char hchar) { + if (hchar >= '0' && hchar <= '9') { + return (ord(hchar)-ord('0')); + } else { return (10+ord(hchar)-ord('A')); - } + } } ''' for func in c_function.searchString(source_code): print("%(name)s (%(type)s) args: %(args)s" % func) + prints:: + is_odd (int) args: [['int', 'x']] dec_to_hex (int) args: [['char', 'hchar']] """ @@ -5226,7 +5753,7 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop ).setParseAction(lambda t:t[0].strip())) else: if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr + + content = (Combine(OneOrMore(~ignoreExpr + ~Literal(opener) + ~Literal(closer) + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) ).setParseAction(lambda t:t[0].strip())) @@ -5245,23 +5772,24 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop return ret def indentedBlock(blockStatementExpr, indentStack, indent=True): - """ - Helper method for defining space-delimited indentation blocks, such as - those used to define block statements in Python source code. + """Helper method for defining space-delimited indentation blocks, + such as those used to define block statements in Python source code. Parameters: - - blockStatementExpr - expression defining syntax of statement that - is repeated within the indented block - - indentStack - list created by caller to manage indentation stack - (multiple statementWithIndentedBlock expressions within a single grammar - should share a common indentStack) - - indent - boolean indicating whether block must be indented beyond the - the current level; set to False for block of left-most statements - (default=C{True}) - A valid block must contain at least one C{blockStatement}. + - blockStatementExpr - expression defining syntax of statement that + is repeated within the indented block + - indentStack - list created by caller to manage indentation stack + (multiple statementWithIndentedBlock expressions within a single + grammar should share a common indentStack) + - indent - boolean indicating whether block must be indented beyond + the the current level; set to False for block of left-most + statements (default= ``True``) + + A valid block must contain at least one ``blockStatement``. Example:: + data = ''' def A(z): A1 @@ -5302,7 +5830,9 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): parseTree = module_body.parseString(data) parseTree.pprint() + prints:: + [['def', 'A', ['(', 'z', ')'], @@ -5320,7 +5850,7 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): 'spam', ['(', 'x', 'y', ')'], ':', - [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] + [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] """ def checkPeerIndent(s,l,t): if l >= len(s): return @@ -5370,51 +5900,61 @@ def replaceHTMLEntity(t): # it's easy to get these comment structures wrong - they're very common, so may as well make them available cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment") -"Comment of the form C{/* ... */}" +"Comment of the form ``/* ... */``" htmlComment = Regex(r"").setName("HTML comment") -"Comment of the form C{}" +"Comment of the form ````" restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line") dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment") -"Comment of the form C{// ... (to end of line)}" +"Comment of the form ``// ... (to end of line)``" cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment") -"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}" +"Comment of either form :class:`cStyleComment` or :class:`dblSlashComment`" javaStyleComment = cppStyleComment -"Same as C{L{cppStyleComment}}" +"Same as :class:`cppStyleComment`" pythonStyleComment = Regex(r"#.*").setName("Python style comment") -"Comment of the form C{# ... (to end of line)}" +"Comment of the form ``# ... (to end of line)``" _commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') + Optional( Word(" \t") + ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem") commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList") -"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas. - This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}.""" +"""(Deprecated) Predefined expression of 1 or more printable words or +quoted strings, separated by commas. + +This expression is deprecated in favor of :class:`pyparsing_common.comma_separated_list`. +""" # some other useful expressions - using lower-case class name since we are really using this as a namespace class pyparsing_common: - """ - Here are some common low-level expressions that may be useful in jump-starting parser development: - - numeric forms (L{integers}, L{reals}, L{scientific notation}) - - common L{programming identifiers} - - network addresses (L{MAC}, L{IPv4}, L{IPv6}) - - ISO8601 L{dates} and L{datetime} - - L{UUID} - - L{comma-separated list} + """Here are some common low-level expressions that may be useful in + jump-starting parser development: + + - numeric forms (:class:`integers`, :class:`reals`, + :class:`scientific notation`) + - common :class:`programming identifiers` + - network addresses (:class:`MAC`, + :class:`IPv4`, :class:`IPv6`) + - ISO8601 :class:`dates` and + :class:`datetime` + - :class:`UUID` + - :class:`comma-separated list` + Parse actions: - - C{L{convertToInteger}} - - C{L{convertToFloat}} - - C{L{convertToDate}} - - C{L{convertToDatetime}} - - C{L{stripHTMLTags}} - - C{L{upcaseTokens}} - - C{L{downcaseTokens}} + + - :class:`convertToInteger` + - :class:`convertToFloat` + - :class:`convertToDate` + - :class:`convertToDatetime` + - :class:`stripHTMLTags` + - :class:`upcaseTokens` + - :class:`downcaseTokens` Example:: + pyparsing_common.number.runTests(''' # any int or real number, returned as the appropriate type 100 @@ -5461,7 +6001,9 @@ class pyparsing_common: # uuid 12345678-1234-5678-1234-567812345678 ''') + prints:: + # any int or real number, returned as the appropriate type 100 [100] @@ -5563,7 +6105,8 @@ class pyparsing_common: """expression that parses a floating point number and returns a float""" sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) - """expression that parses a floating point number with optional scientific notation and returns a float""" + """expression that parses a floating point number with optional + scientific notation and returns a float""" # streamlining this expression makes the docs nicer-looking number = (sci_real | real | signed_integer).streamline() @@ -5571,12 +6114,12 @@ class pyparsing_common: fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat) """any int or real number, returned as float""" - + identifier = Word(alphas+'_', alphanums+'_').setName("identifier") """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')""" - + ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address") - "IPv4 address (C{0.0.0.0 - 255.255.255.255})" + "IPv4 address (``0.0.0.0 - 255.255.255.255``)" _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer") _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address") @@ -5585,7 +6128,7 @@ class pyparsing_common: _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address") ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address") "IPv6 address (long, short, or mixed form)" - + mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address") "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)" @@ -5595,13 +6138,16 @@ class pyparsing_common: Helper to create a parse action for converting parsed date string to Python datetime.date Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%d"``) Example:: + date_expr = pyparsing_common.iso8601_date.copy() date_expr.setParseAction(pyparsing_common.convertToDate()) print(date_expr.parseString("1999-12-31")) + prints:: + [datetime.date(1999, 12, 31)] """ def cvt_fn(s,l,t): @@ -5613,17 +6159,20 @@ class pyparsing_common: @staticmethod def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"): - """ - Helper to create a parse action for converting parsed datetime string to Python datetime.datetime + """Helper to create a parse action for converting parsed + datetime string to Python datetime.datetime Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%dT%H:%M:%S.%f"``) Example:: + dt_expr = pyparsing_common.iso8601_datetime.copy() dt_expr.setParseAction(pyparsing_common.convertToDatetime()) print(dt_expr.parseString("1999-12-31T23:59:59.999")) + prints:: + [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)] """ def cvt_fn(s,l,t): @@ -5634,31 +6183,34 @@ class pyparsing_common: return cvt_fn iso8601_date = Regex(r'(?P\d{4})(?:-(?P\d\d)(?:-(?P\d\d))?)?').setName("ISO8601 date") - "ISO8601 date (C{yyyy-mm-dd})" + "ISO8601 date (``yyyy-mm-dd``)" iso8601_datetime = Regex(r'(?P\d{4})-(?P\d\d)-(?P\d\d)[T ](?P\d\d):(?P\d\d)(:(?P\d\d(\.\d*)?)?)?(?PZ|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime") - "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}" + "ISO8601 datetime (``yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)``) - trailing seconds, milliseconds, and timezone optional; accepts separating ``'T'`` or ``' '``" uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID") - "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})" + "UUID (``xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx``)" _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress() @staticmethod def stripHTMLTags(s, l, tokens): - """ - Parse action to remove HTML tags from web page HTML source + """Parse action to remove HTML tags from web page HTML source Example:: - # strip HTML links from normal text - text = 'More info at the pyparsing wiki page' + + # strip HTML links from normal text + text = 'More info at the pyparsing wiki page' td,td_end = makeHTMLTags("TD") table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end - - print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page' + print(table_text.parseString(text).body) + + Prints:: + + More info at the pyparsing wiki page """ return pyparsing_common._html_stripper.transformString(tokens[0]) - _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') + _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') + Optional( White(" \t") ) ) ).streamline().setName("commaItem") comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list") """Predefined expression of 1 or more printable words or quoted strings, separated by commas.""" @@ -5670,6 +6222,164 @@ class pyparsing_common: """Parse action to convert tokens to lower case.""" +class _lazyclassproperty(object): + def __init__(self, fn): + self.fn = fn + self.__doc__ = fn.__doc__ + self.__name__ = fn.__name__ + + def __get__(self, obj, cls): + if cls is None: + cls = type(obj) + if not hasattr(cls, '_intern') or any(cls._intern is getattr(superclass, '_intern', []) for superclass in cls.__mro__[1:]): + cls._intern = {} + attrname = self.fn.__name__ + if attrname not in cls._intern: + cls._intern[attrname] = self.fn(cls) + return cls._intern[attrname] + + +class unicode_set(object): + """ + A set of Unicode characters, for language-specific strings for + ``alphas``, ``nums``, ``alphanums``, and ``printables``. + A unicode_set is defined by a list of ranges in the Unicode character + set, in a class attribute ``_ranges``, such as:: + + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + A unicode set can also be defined using multiple inheritance of other unicode sets:: + + class CJK(Chinese, Japanese, Korean): + pass + """ + _ranges = [] + + @classmethod + def _get_chars_for_ranges(cls): + ret = [] + for cc in cls.__mro__: + if cc is unicode_set: + break + for rr in cc._ranges: + ret.extend(range(rr[0], rr[-1]+1)) + return [unichr(c) for c in sorted(set(ret))] + + @_lazyclassproperty + def printables(cls): + "all non-whitespace characters in this range" + return u''.join(filterfalse(unicode.isspace, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphas(cls): + "all alphabetic characters in this range" + return u''.join(filter(unicode.isalpha, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def nums(cls): + "all numeric digit characters in this range" + return u''.join(filter(unicode.isdigit, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphanums(cls): + "all alphanumeric characters in this range" + return cls.alphas + cls.nums + + +class pyparsing_unicode(unicode_set): + """ + A namespace class for defining common language unicode_sets. + """ + _ranges = [(32, sys.maxunicode)] + + class Latin1(unicode_set): + "Unicode set for Latin-1 Unicode Character Range" + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + class LatinA(unicode_set): + "Unicode set for Latin-A Unicode Character Range" + _ranges = [(0x0100, 0x017f),] + + class LatinB(unicode_set): + "Unicode set for Latin-B Unicode Character Range" + _ranges = [(0x0180, 0x024f),] + + class Greek(unicode_set): + "Unicode set for Greek Unicode Character Ranges" + _ranges = [ + (0x0370, 0x03ff), (0x1f00, 0x1f15), (0x1f18, 0x1f1d), (0x1f20, 0x1f45), (0x1f48, 0x1f4d), + (0x1f50, 0x1f57), (0x1f59,), (0x1f5b,), (0x1f5d,), (0x1f5f, 0x1f7d), (0x1f80, 0x1fb4), (0x1fb6, 0x1fc4), + (0x1fc6, 0x1fd3), (0x1fd6, 0x1fdb), (0x1fdd, 0x1fef), (0x1ff2, 0x1ff4), (0x1ff6, 0x1ffe), + ] + + class Cyrillic(unicode_set): + "Unicode set for Cyrillic Unicode Character Range" + _ranges = [(0x0400, 0x04ff)] + + class Chinese(unicode_set): + "Unicode set for Chinese Unicode Character Range" + _ranges = [(0x4e00, 0x9fff), (0x3000, 0x303f), ] + + class Japanese(unicode_set): + "Unicode set for Japanese Unicode Character Range, combining Kanji, Hiragana, and Katakana ranges" + _ranges = [ ] + + class Kanji(unicode_set): + "Unicode set for Kanji Unicode Character Range" + _ranges = [(0x4E00, 0x9Fbf), (0x3000, 0x303f), ] + + class Hiragana(unicode_set): + "Unicode set for Hiragana Unicode Character Range" + _ranges = [(0x3040, 0x309f), ] + + class Katakana(unicode_set): + "Unicode set for Katakana Unicode Character Range" + _ranges = [(0x30a0, 0x30ff), ] + + class Korean(unicode_set): + "Unicode set for Korean Unicode Character Range" + _ranges = [(0xac00, 0xd7af), (0x1100, 0x11ff), (0x3130, 0x318f), (0xa960, 0xa97f), (0xd7b0, 0xd7ff), (0x3000, 0x303f), ] + + class CJK(Chinese, Japanese, Korean): + "Unicode set for combined Chinese, Japanese, and Korean (CJK) Unicode Character Range" + pass + + class Thai(unicode_set): + "Unicode set for Thai Unicode Character Range" + _ranges = [(0x0e01, 0x0e3a), (0x0e3f, 0x0e5b), ] + + class Arabic(unicode_set): + "Unicode set for Arabic Unicode Character Range" + _ranges = [(0x0600, 0x061b), (0x061e, 0x06ff), (0x0700, 0x077f), ] + + class Hebrew(unicode_set): + "Unicode set for Hebrew Unicode Character Range" + _ranges = [(0x0590, 0x05ff), ] + + class Devanagari(unicode_set): + "Unicode set for Devanagari Unicode Character Range" + _ranges = [(0x0900, 0x097f), (0xa8e0, 0xa8ff)] + +pyparsing_unicode.Japanese._ranges = (pyparsing_unicode.Japanese.Kanji._ranges + + pyparsing_unicode.Japanese.Hiragana._ranges + + pyparsing_unicode.Japanese.Katakana._ranges) + +# define ranges in language character sets +if PY_3: + setattr(pyparsing_unicode, "العربية", pyparsing_unicode.Arabic) + setattr(pyparsing_unicode, "中文", pyparsing_unicode.Chinese) + setattr(pyparsing_unicode, "кириллица", pyparsing_unicode.Cyrillic) + setattr(pyparsing_unicode, "Ελληνικά", pyparsing_unicode.Greek) + setattr(pyparsing_unicode, "עִברִית", pyparsing_unicode.Hebrew) + setattr(pyparsing_unicode, "日本語", pyparsing_unicode.Japanese) + setattr(pyparsing_unicode.Japanese, "漢字", pyparsing_unicode.Japanese.Kanji) + setattr(pyparsing_unicode.Japanese, "カタカナ", pyparsing_unicode.Japanese.Katakana) + setattr(pyparsing_unicode.Japanese, "ひらがな", pyparsing_unicode.Japanese.Hiragana) + setattr(pyparsing_unicode, "한국어", pyparsing_unicode.Korean) + setattr(pyparsing_unicode, "ไทย", pyparsing_unicode.Thai) + setattr(pyparsing_unicode, "देवनागरी", pyparsing_unicode.Devanagari) + + if __name__ == "__main__": selectToken = CaselessLiteral("select") @@ -5683,7 +6393,7 @@ if __name__ == "__main__": tableName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens) tableNameList = Group(delimitedList(tableName)).setName("tables") - + simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables") # demo runTests method, including embedded comments in test string diff --git a/pipenv/patched/notpip/_vendor/pytoml/__init__.py b/pipenv/patched/notpip/_vendor/pytoml/__init__.py index 8dc73155..8ed060ff 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/__init__.py +++ b/pipenv/patched/notpip/_vendor/pytoml/__init__.py @@ -1,3 +1,4 @@ from .core import TomlError from .parser import load, loads -from .writer import dump, dumps +from .test import translate_to_test +from .writer import dump, dumps \ No newline at end of file diff --git a/pipenv/patched/notpip/_vendor/pytoml/parser.py b/pipenv/patched/notpip/_vendor/pytoml/parser.py index 9f94e923..3493aa64 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/parser.py +++ b/pipenv/patched/notpip/_vendor/pytoml/parser.py @@ -1,5 +1,6 @@ import string, re, sys, datetime from .core import TomlError +from .utils import rfc3339_re, parse_rfc3339_re if sys.version_info[0] == 2: _chr = unichr @@ -179,13 +180,13 @@ _ws_re = re.compile(r'[ \t]*') def _p_ws(s): s.expect_re(_ws_re) -_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', '\'': '\'', - '\\': '\\', '/': '/', 'f': '\f' } +_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', + '\\': '\\', 'f': '\f' } _basicstr_re = re.compile(r'[^"\\\000-\037]*') _short_uni_re = re.compile(r'u([0-9a-fA-F]{4})') _long_uni_re = re.compile(r'U([0-9a-fA-F]{8})') -_escapes_re = re.compile('[bnrt"\'\\\\/f]') +_escapes_re = re.compile(r'[btnfr\"\\]') _newline_esc_re = re.compile('\n[ \t\n]*') def _p_basicstr_content(s, content=_basicstr_re): res = [] @@ -196,7 +197,10 @@ def _p_basicstr_content(s, content=_basicstr_re): if s.consume_re(_newline_esc_re): pass elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re): - res.append(_chr(int(s.last().group(1), 16))) + v = int(s.last().group(1), 16) + if 0xd800 <= v < 0xe000: + s.fail() + res.append(_chr(v)) else: s.expect_re(_escapes_re) res.append(_escapes[s.last().group(0)]) @@ -220,9 +224,8 @@ def _p_key(s): return s.expect_re(_key_re).group(0) _float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?') -_datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') -_basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') +_basicstr_ml_re = re.compile(r'(?:""?(?!")|[^"\\\000-\011\013-\037])*') _litstr_re = re.compile(r"[^'\000\010\012-\037]*") _litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") def _p_value(s, object_pairs_hook): @@ -251,24 +254,9 @@ def _p_value(s, object_pairs_hook): s.expect('\'') return 'str', r, r, pos - if s.consume_re(_datetime_re): + if s.consume_re(rfc3339_re): m = s.last() - s0 = m.group(0) - r = map(int, m.groups()[:6]) - if m.group(7): - micro = float(m.group(7)) - else: - micro = 0 - - if m.group(8): - g = int(m.group(8), 10) * 60 + int(m.group(9), 10) - tz = _TimeZone(datetime.timedelta(0, g * 60)) - else: - tz = _TimeZone(datetime.timedelta(0, 0)) - - y, m, d, H, M, S = r - dt = datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) - return 'datetime', s0, dt, pos + return 'datetime', m.group(0), parse_rfc3339_re(m), pos if s.consume_re(_float_re): m = s.last().group(0) @@ -351,24 +339,3 @@ def _p_toml(s, object_pairs_hook): _p_ews(s) s.expect_eof() return stmts - -class _TimeZone(datetime.tzinfo): - def __init__(self, offset): - self._offset = offset - - def utcoffset(self, dt): - return self._offset - - def dst(self, dt): - return None - - def tzname(self, dt): - m = self._offset.total_seconds() // 60 - if m < 0: - res = '-' - m = -m - else: - res = '+' - h = m // 60 - m = m - h * 60 - return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/pipenv/patched/notpip/_vendor/pytoml/test.py b/pipenv/patched/notpip/_vendor/pytoml/test.py new file mode 100644 index 00000000..ec8abfc6 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pytoml/test.py @@ -0,0 +1,30 @@ +import datetime +from .utils import format_rfc3339 + +try: + _string_types = (str, unicode) + _int_types = (int, long) +except NameError: + _string_types = str + _int_types = int + +def translate_to_test(v): + if isinstance(v, dict): + return { k: translate_to_test(v) for k, v in v.items() } + if isinstance(v, list): + a = [translate_to_test(x) for x in v] + if v and isinstance(v[0], dict): + return a + else: + return {'type': 'array', 'value': a} + if isinstance(v, datetime.datetime): + return {'type': 'datetime', 'value': format_rfc3339(v)} + if isinstance(v, bool): + return {'type': 'bool', 'value': 'true' if v else 'false'} + if isinstance(v, _int_types): + return {'type': 'integer', 'value': str(v)} + if isinstance(v, float): + return {'type': 'float', 'value': '{:.17}'.format(v)} + if isinstance(v, _string_types): + return {'type': 'string', 'value': v} + raise RuntimeError('unexpected value: {!r}'.format(v)) diff --git a/pipenv/patched/notpip/_vendor/pytoml/utils.py b/pipenv/patched/notpip/_vendor/pytoml/utils.py new file mode 100644 index 00000000..636a680b --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pytoml/utils.py @@ -0,0 +1,67 @@ +import datetime +import re + +rfc3339_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') + +def parse_rfc3339(v): + m = rfc3339_re.match(v) + if not m or m.group(0) != v: + return None + return parse_rfc3339_re(m) + +def parse_rfc3339_re(m): + r = map(int, m.groups()[:6]) + if m.group(7): + micro = float(m.group(7)) + else: + micro = 0 + + if m.group(8): + g = int(m.group(8), 10) * 60 + int(m.group(9), 10) + tz = _TimeZone(datetime.timedelta(0, g * 60)) + else: + tz = _TimeZone(datetime.timedelta(0, 0)) + + y, m, d, H, M, S = r + return datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) + + +def format_rfc3339(v): + offs = v.utcoffset() + offs = int(offs.total_seconds()) // 60 if offs is not None else 0 + + if offs == 0: + suffix = 'Z' + else: + if offs > 0: + suffix = '+' + else: + suffix = '-' + offs = -offs + suffix = '{0}{1:02}:{2:02}'.format(suffix, offs // 60, offs % 60) + + if v.microsecond: + return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix + else: + return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix + +class _TimeZone(datetime.tzinfo): + def __init__(self, offset): + self._offset = offset + + def utcoffset(self, dt): + return self._offset + + def dst(self, dt): + return None + + def tzname(self, dt): + m = self._offset.total_seconds() // 60 + if m < 0: + res = '-' + m = -m + else: + res = '+' + h = m // 60 + m = m - h * 60 + return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/pipenv/patched/notpip/_vendor/pytoml/writer.py b/pipenv/patched/notpip/_vendor/pytoml/writer.py index 6eaf5d76..73b5089c 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/writer.py +++ b/pipenv/patched/notpip/_vendor/pytoml/writer.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals -import io, datetime, math, sys +import io, datetime, math, string, sys + +from .utils import format_rfc3339 if sys.version_info[0] == 3: long = int @@ -39,22 +41,13 @@ def _escape_string(s): return '"' + ''.join(res) + '"' +_key_chars = string.digits + string.ascii_letters + '-_' def _escape_id(s): - if any(not c.isalnum() and c not in '-_' for c in s): + if any(c not in _key_chars for c in s): return _escape_string(s) return s -def _format_list(v): - return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) - -# Formula from: -# https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds -# Once support for py26 is dropped, this can be replaced by td.total_seconds() -def _total_seconds(td): - return ((td.microseconds - + (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6) - def _format_value(v): if isinstance(v, bool): return 'true' if v else 'false' @@ -68,25 +61,11 @@ def _format_value(v): elif isinstance(v, unicode) or isinstance(v, bytes): return _escape_string(v) elif isinstance(v, datetime.datetime): - offs = v.utcoffset() - offs = _total_seconds(offs) // 60 if offs is not None else 0 - - if offs == 0: - suffix = 'Z' - else: - if offs > 0: - suffix = '+' - else: - suffix = '-' - offs = -offs - suffix = '{0}{1:.02}{2:.02}'.format(suffix, offs // 60, offs % 60) - - if v.microsecond: - return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix - else: - return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix + return format_rfc3339(v) elif isinstance(v, list): - return _format_list(v) + return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) + elif isinstance(v, dict): + return '{{{0}}}'.format(', '.join('{} = {}'.format(_escape_id(k), _format_value(obj)) for k, obj in v.items())) else: raise RuntimeError(v) diff --git a/pipenv/patched/notpip/_vendor/requests/__init__.py b/pipenv/patched/notpip/_vendor/requests/__init__.py index af3d4b1d..1544bc89 100644 --- a/pipenv/patched/notpip/_vendor/requests/__init__.py +++ b/pipenv/patched/notpip/_vendor/requests/__init__.py @@ -22,7 +22,7 @@ usage: ... or POST: >>> payload = dict(key1='value1', key2='value2') - >>> r = requests.post('http://httpbin.org/post', data=payload) + >>> r = requests.post('https://httpbin.org/post', data=payload) >>> print(r.text) { ... @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.23 + # urllib3 >= 1.21.1, <= 1.24 assert major == 1 assert minor >= 21 - assert minor <= 23 + assert minor <= 24 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] @@ -79,14 +79,14 @@ def _check_cryptography(cryptography_version): return if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warning = 'Old version of cryptography ({}) may cause slowdown.'.format(cryptography_version) warnings.warn(warning, RequestsDependencyWarning) # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) except (AssertionError, ValueError): - warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported " + warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " "version!".format(urllib3.__version__, chardet.__version__), RequestsDependencyWarning) @@ -125,12 +125,7 @@ from .exceptions import ( # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/pipenv/patched/notpip/_vendor/requests/__version__.py b/pipenv/patched/notpip/_vendor/requests/__version__.py index ef61ec0f..f5b5d036 100644 --- a/pipenv/patched/notpip/_vendor/requests/__version__.py +++ b/pipenv/patched/notpip/_vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.19.1' -__build__ = 0x021901 +__version__ = '2.21.0' +__build__ = 0x022100 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/pipenv/patched/notpip/_vendor/requests/adapters.py b/pipenv/patched/notpip/_vendor/requests/adapters.py index 014c2675..abae6d6d 100644 --- a/pipenv/patched/notpip/_vendor/requests/adapters.py +++ b/pipenv/patched/notpip/_vendor/requests/adapters.py @@ -26,6 +26,7 @@ from pipenv.patched.notpip._vendor.urllib3.exceptions import ProtocolError from pipenv.patched.notpip._vendor.urllib3.exceptions import ReadTimeoutError from pipenv.patched.notpip._vendor.urllib3.exceptions import SSLError as _SSLError from pipenv.patched.notpip._vendor.urllib3.exceptions import ResponseError +from pipenv.patched.notpip._vendor.urllib3.exceptions import LocationValueError from .models import Response from .compat import urlparse, basestring @@ -35,7 +36,8 @@ from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL, + InvalidURL) from .auth import _basic_auth_str try: @@ -127,8 +129,7 @@ class HTTPAdapter(BaseAdapter): self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in - self.__attrs__) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because @@ -224,7 +225,7 @@ class HTTPAdapter(BaseAdapter): if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) + "invalid path: {}".format(cert_loc)) conn.cert_reqs = 'CERT_REQUIRED' @@ -246,10 +247,10 @@ class HTTPAdapter(BaseAdapter): conn.key_file = None if conn.cert_file and not os.path.exists(conn.cert_file): raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(conn.cert_file)) + "invalid path: {}".format(conn.cert_file)) if conn.key_file and not os.path.exists(conn.key_file): raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(conn.key_file)) + "invalid path: {}".format(conn.key_file)) def build_response(self, req, resp): """Builds a :class:`Response ` object from a urllib3 @@ -378,7 +379,7 @@ class HTTPAdapter(BaseAdapter): when subclassing the :class:`HTTPAdapter `. - :param proxies: The url of the proxy being used for this request. + :param proxy: The url of the proxy being used for this request. :rtype: dict """ headers = {} @@ -407,7 +408,10 @@ class HTTPAdapter(BaseAdapter): :rtype: requests.Response """ - conn = self.get_connection(request.url, proxies) + try: + conn = self.get_connection(request.url, proxies) + except LocationValueError as e: + raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) @@ -421,7 +425,7 @@ class HTTPAdapter(BaseAdapter): timeout = TimeoutSauce(connect=connect, read=read) except ValueError as e: # this may raise a string formatting error. - err = ("Invalid timeout {0}. Pass a (connect, read) " + err = ("Invalid timeout {}. Pass a (connect, read) " "timeout tuple, or a single float to set " "both timeouts to the same value".format(timeout)) raise ValueError(err) @@ -471,11 +475,10 @@ class HTTPAdapter(BaseAdapter): # Receive the response from the server try: - # For Python 2.7+ versions, use buffering of HTTP - # responses + # For Python 2.7, use buffering of HTTP responses r = low_conn.getresponse(buffering=True) except TypeError: - # For compatibility with Python 2.6 versions and back + # For compatibility with Python 3.3+ r = low_conn.getresponse() resp = HTTPResponse.from_httplib( diff --git a/pipenv/patched/notpip/_vendor/requests/api.py b/pipenv/patched/notpip/_vendor/requests/api.py index a2cc84d7..abada96d 100644 --- a/pipenv/patched/notpip/_vendor/requests/api.py +++ b/pipenv/patched/notpip/_vendor/requests/api.py @@ -18,8 +18,10 @@ def request(method, url, **kwargs): :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. @@ -47,7 +49,7 @@ def request(method, url, **kwargs): Usage:: >>> import requests - >>> req = requests.request('GET', 'http://httpbin.org/get') + >>> req = requests.request('GET', 'https://httpbin.org/get') """ @@ -62,7 +64,8 @@ def get(url, params=None, **kwargs): r"""Sends a GET request. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response @@ -102,7 +105,8 @@ def post(url, data=None, json=None, **kwargs): r"""Sends a POST request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -116,7 +120,8 @@ def put(url, data=None, **kwargs): r"""Sends a PUT request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -130,7 +135,8 @@ def patch(url, data=None, **kwargs): r"""Sends a PATCH request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object diff --git a/pipenv/patched/notpip/_vendor/requests/auth.py b/pipenv/patched/notpip/_vendor/requests/auth.py index 4ae45947..bdde51c7 100644 --- a/pipenv/patched/notpip/_vendor/requests/auth.py +++ b/pipenv/patched/notpip/_vendor/requests/auth.py @@ -38,7 +38,7 @@ def _basic_auth_str(username, password): if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(username), category=DeprecationWarning, @@ -48,7 +48,7 @@ def _basic_auth_str(username, password): if not isinstance(password, basestring): warnings.warn( "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(password), category=DeprecationWarning, diff --git a/pipenv/patched/notpip/_vendor/requests/compat.py b/pipenv/patched/notpip/_vendor/requests/compat.py index 4fbd6231..7c143940 100644 --- a/pipenv/patched/notpip/_vendor/requests/compat.py +++ b/pipenv/patched/notpip/_vendor/requests/compat.py @@ -47,9 +47,8 @@ if is_py2: import cookielib from Cookie import Morsel from StringIO import StringIO - from collections import Callable, Mapping, MutableMapping + from collections import Callable, Mapping, MutableMapping, OrderedDict - from pipenv.patched.notpip._vendor.urllib3.packages.ordered_dict import OrderedDict builtin_str = str bytes = str diff --git a/pipenv/patched/notpip/_vendor/requests/cookies.py b/pipenv/patched/notpip/_vendor/requests/cookies.py index 50883a84..56fccd9c 100644 --- a/pipenv/patched/notpip/_vendor/requests/cookies.py +++ b/pipenv/patched/notpip/_vendor/requests/cookies.py @@ -444,20 +444,21 @@ def create_cookie(name, value, **kwargs): By default, the pair of `name` and `value` will be set for the domain '' and sent on every request (this is sometimes called a "supercookie"). """ - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False,) + result = { + 'version': 0, + 'name': name, + 'value': value, + 'port': None, + 'domain': '', + 'path': '/', + 'secure': False, + 'expires': None, + 'discard': True, + 'comment': None, + 'comment_url': None, + 'rest': {'HttpOnly': None}, + 'rfc2109': False, + } badargs = set(kwargs) - set(result) if badargs: @@ -511,6 +512,7 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): :param cookiejar: (optional) A cookiejar to add the cookies to. :param overwrite: (optional) If False, will not replace cookies already in the jar with new ones. + :rtype: CookieJar """ if cookiejar is None: cookiejar = RequestsCookieJar() @@ -529,6 +531,7 @@ def merge_cookies(cookiejar, cookies): :param cookiejar: CookieJar object to add the cookies to. :param cookies: Dictionary or CookieJar object to be added. + :rtype: CookieJar """ if not isinstance(cookiejar, cookielib.CookieJar): raise ValueError('You can only merge into CookieJar') diff --git a/pipenv/patched/notpip/_vendor/requests/help.py b/pipenv/patched/notpip/_vendor/requests/help.py index eba69edb..72d72160 100644 --- a/pipenv/patched/notpip/_vendor/requests/help.py +++ b/pipenv/patched/notpip/_vendor/requests/help.py @@ -89,8 +89,7 @@ def info(): 'version': getattr(idna, '__version__', ''), } - # OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module. - system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None) + system_ssl = ssl.OPENSSL_VERSION_NUMBER system_ssl_info = { 'version': '%x' % system_ssl if system_ssl is not None else '' } diff --git a/pipenv/patched/notpip/_vendor/requests/hooks.py b/pipenv/patched/notpip/_vendor/requests/hooks.py index 32b32de7..7a51f212 100644 --- a/pipenv/patched/notpip/_vendor/requests/hooks.py +++ b/pipenv/patched/notpip/_vendor/requests/hooks.py @@ -15,14 +15,14 @@ HOOKS = ['response'] def default_hooks(): - return dict((event, []) for event in HOOKS) + return {event: [] for event in HOOKS} # TODO: response is the only one def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" - hooks = hooks or dict() + hooks = hooks or {} hooks = hooks.get(key) if hooks: if hasattr(hooks, '__call__'): diff --git a/pipenv/patched/notpip/_vendor/requests/models.py b/pipenv/patched/notpip/_vendor/requests/models.py index 6708f09b..5f899c42 100644 --- a/pipenv/patched/notpip/_vendor/requests/models.py +++ b/pipenv/patched/notpip/_vendor/requests/models.py @@ -204,9 +204,13 @@ class Request(RequestHooksMixin): :param url: URL to send. :param headers: dictionary of headers to send. :param files: dictionary of {filename: fileobject} files to multipart upload. - :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. + :param data: the body to attach to the request. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param json: json for the body to attach to the request (if files or data is not specified). - :param params: dictionary of URL parameters to append to the URL. + :param params: URL parameters to append to the URL. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param auth: Auth handler or (user, pass) tuple. :param cookies: dictionary or CookieJar of cookies to attach to this request. :param hooks: dictionary of callback hooks, for internal usage. @@ -214,7 +218,7 @@ class Request(RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> req.prepare() """ @@ -274,7 +278,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> r = req.prepare() @@ -648,10 +652,7 @@ class Response(object): if not self._content_consumed: self.content - return dict( - (attr, getattr(self, attr, None)) - for attr in self.__attrs__ - ) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): @@ -780,7 +781,7 @@ class Response(object): return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. diff --git a/pipenv/patched/notpip/_vendor/requests/sessions.py b/pipenv/patched/notpip/_vendor/requests/sessions.py index ba135268..d73d700f 100644 --- a/pipenv/patched/notpip/_vendor/requests/sessions.py +++ b/pipenv/patched/notpip/_vendor/requests/sessions.py @@ -19,7 +19,7 @@ from .cookies import ( from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook from ._internal_utils import to_native_string -from .utils import to_key_val_list, default_headers +from .utils import to_key_val_list, default_headers, DEFAULT_PORTS from .exceptions import ( TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) @@ -115,6 +115,31 @@ class SessionRedirectMixin(object): return to_native_string(location, 'utf8') return None + def should_strip_auth(self, old_url, new_url): + """Decide whether Authorization header should be removed when redirecting""" + old_parsed = urlparse(old_url) + new_parsed = urlparse(new_url) + if old_parsed.hostname != new_parsed.hostname: + return True + # Special case: allow http -> https redirect when using the standard + # ports. This isn't specified by RFC 7235, but is kept to avoid + # breaking backwards compatibility with older versions of requests + # that allowed any redirects on the same host. + if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) + and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): + return False + + # Handle default port usage corresponding to scheme. + changed_port = old_parsed.port != new_parsed.port + changed_scheme = old_parsed.scheme != new_parsed.scheme + default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None) + if (not changed_scheme and old_parsed.port in default_port + and new_parsed.port in default_port): + return False + + # Standard case: root URI must match + return changed_port or changed_scheme + def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): """Receives a Response. Returns a generator of Responses or Requests.""" @@ -236,14 +261,10 @@ class SessionRedirectMixin(object): headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: + if 'Authorization' in headers and self.should_strip_auth(response.request.url, url): # If we get redirected to a new host, we should strip out any # authentication headers. - original_parsed = urlparse(response.request.url) - redirect_parsed = urlparse(url) - - if (original_parsed.hostname != redirect_parsed.hostname): - del headers['Authorization'] + del headers['Authorization'] # .netrc might have more auth for us on our new host. new_auth = get_netrc_auth(url) if self.trust_env else None @@ -299,7 +320,7 @@ class SessionRedirectMixin(object): """ method = prepared_request.method - # http://tools.ietf.org/html/rfc7231#section-6.4.4 + # https://tools.ietf.org/html/rfc7231#section-6.4.4 if response.status_code == codes.see_other and method != 'HEAD': method = 'GET' @@ -325,13 +346,13 @@ class Session(SessionRedirectMixin): >>> import requests >>> s = requests.Session() - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') Or as a context manager:: >>> with requests.Session() as s: - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') """ @@ -453,8 +474,8 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary, bytes, or file-like object to send - in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the @@ -550,7 +571,8 @@ class Session(SessionRedirectMixin): r"""Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response @@ -562,7 +584,8 @@ class Session(SessionRedirectMixin): r"""Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -573,7 +596,8 @@ class Session(SessionRedirectMixin): r"""Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -723,7 +747,7 @@ class Session(SessionRedirectMixin): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state def __setstate__(self, state): @@ -735,7 +759,12 @@ def session(): """ Returns a :class:`Session` for context-management. + .. deprecated:: 1.0.0 + + This method has been deprecated since version 1.0.0 and is only kept for + backwards compatibility. New code should use :class:`~requests.sessions.Session` + to create a session. This may be removed at a future date. + :rtype: Session """ - return Session() diff --git a/pipenv/patched/notpip/_vendor/requests/status_codes.py b/pipenv/patched/notpip/_vendor/requests/status_codes.py index ff462c6c..813e8c4e 100644 --- a/pipenv/patched/notpip/_vendor/requests/status_codes.py +++ b/pipenv/patched/notpip/_vendor/requests/status_codes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" +r""" The ``codes`` object defines a mapping from common names for HTTP statuses to their numerical codes, accessible either as attributes or as dictionary items. diff --git a/pipenv/patched/notpip/_vendor/requests/utils.py b/pipenv/patched/notpip/_vendor/requests/utils.py index 431f6be0..8170a8d2 100644 --- a/pipenv/patched/notpip/_vendor/requests/utils.py +++ b/pipenv/patched/notpip/_vendor/requests/utils.py @@ -38,6 +38,8 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() +DEFAULT_PORTS = {'http': 80, 'https': 443} + if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups @@ -173,10 +175,10 @@ def get_netrc_auth(url, raise_errors=False): for f in NETRC_FILES: try: - loc = os.path.expanduser('~/{0}'.format(f)) + loc = os.path.expanduser('~/{}'.format(f)) except KeyError: # os.path.expanduser can fail when $HOME is undefined and - # getpwuid fails. See http://bugs.python.org/issue20164 & + # getpwuid fails. See https://bugs.python.org/issue20164 & # https://github.com/requests/requests/issues/1846 return @@ -264,7 +266,7 @@ def from_key_val_list(value): >>> from_key_val_list([('key', 'val')]) OrderedDict([('key', 'val')]) >>> from_key_val_list('string') - ValueError: need more than 1 value to unpack + ValueError: cannot encode objects that are not 2-tuples >>> from_key_val_list({'key': 'val'}) OrderedDict([('key', 'val')]) @@ -466,7 +468,7 @@ def _parse_content_type_header(header): if index_of_equals != -1: key = param[:index_of_equals].strip(items_to_strip) value = param[index_of_equals + 1:].strip(items_to_strip) - params_dict[key] = value + params_dict[key.lower()] = value return content_type, params_dict @@ -706,6 +708,10 @@ def should_bypass_proxies(url, no_proxy): no_proxy = get_proxy('no_proxy') parsed = urlparse(url) + if parsed.hostname is None: + # URLs don't always have hostnames, e.g. file:/// urls. + return True + if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the hostname, both with and without the port. @@ -725,7 +731,7 @@ def should_bypass_proxies(url, no_proxy): else: host_with_port = parsed.hostname if parsed.port: - host_with_port += ':{0}'.format(parsed.port) + host_with_port += ':{}'.format(parsed.port) for host in no_proxy: if parsed.hostname.endswith(host) or host_with_port.endswith(host): @@ -733,13 +739,8 @@ def should_bypass_proxies(url, no_proxy): # to apply the proxies on this URL. return True - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - # The proxy_bypass function is incredibly buggy on OS X in early versions - # of Python 2.6, so allow this call to fail. Only catch the specific - # exceptions we've seen, though: this call failing in other ways can reveal - # legitimate problems. with set_environ('no_proxy', no_proxy_arg): + # parsed.hostname can be `None` in cases such as a file URI. try: bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): diff --git a/pipenv/patched/notpip/_vendor/six.py b/pipenv/patched/notpip/_vendor/six.py index 6bf4fd38..89b2188f 100644 --- a/pipenv/patched/notpip/_vendor/six.py +++ b/pipenv/patched/notpip/_vendor/six.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2017 Benjamin Peterson +# Copyright (c) 2010-2018 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.11.0" +__version__ = "1.12.0" # Useful for very coarse version differentiation. @@ -844,10 +844,71 @@ def add_metaclass(metaclass): orig_vars.pop(slots_var) orig_vars.pop('__dict__', None) orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + + def python_2_unicode_compatible(klass): """ A decorator that defines __unicode__ and __str__ methods under Python 2. diff --git a/pipenv/patched/notpip/_vendor/urllib3/__init__.py b/pipenv/patched/notpip/_vendor/urllib3/__init__.py index 4bd533b5..148a9c31 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/__init__.py +++ b/pipenv/patched/notpip/_vendor/urllib3/__init__.py @@ -23,16 +23,11 @@ from .util.retry import Retry # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.23' +__version__ = '1.24.1' __all__ = ( 'HTTPConnectionPool', diff --git a/pipenv/patched/notpip/_vendor/urllib3/_collections.py b/pipenv/patched/notpip/_vendor/urllib3/_collections.py index 6e36b84e..34f23811 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/_collections.py +++ b/pipenv/patched/notpip/_vendor/urllib3/_collections.py @@ -14,10 +14,7 @@ except ImportError: # Platform-specific: No threads available pass -try: # Python 2.7+ - from collections import OrderedDict -except ImportError: - from .packages.ordered_dict import OrderedDict +from collections import OrderedDict from .exceptions import InvalidHeader from .packages.six import iterkeys, itervalues, PY3 diff --git a/pipenv/patched/notpip/_vendor/urllib3/connection.py b/pipenv/patched/notpip/_vendor/urllib3/connection.py index a03b573f..02b36654 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/connection.py +++ b/pipenv/patched/notpip/_vendor/urllib3/connection.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import datetime import logging import os -import sys import socket from socket import error as SocketError, timeout as SocketTimeout import warnings @@ -78,9 +77,6 @@ class HTTPConnection(_HTTPConnection, object): - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool` - ``source_address``: Set the source address for the current connection. - - .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x - - ``socket_options``: Set specific options on the underlying socket. If not specified, then defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy. @@ -108,21 +104,13 @@ class HTTPConnection(_HTTPConnection, object): if six.PY3: # Python 3 kw.pop('strict', None) - # Pre-set source_address in case we have an older Python like 2.6. + # Pre-set source_address. self.source_address = kw.get('source_address') - if sys.version_info < (2, 7): # Python 2.6 - # _HTTPConnection on Python 2.6 will balk at this keyword arg, but - # not newer versions. We can still use it when creating a - # connection though, so we pop it *after* we have saved it as - # self.source_address. - kw.pop('source_address', None) - #: The socket options provided by the user. If no options are #: provided, we use the default options. self.socket_options = kw.pop('socket_options', self.default_socket_options) - # Superclass also sets self.source_address in Python 2.7+. _HTTPConnection.__init__(self, *args, **kw) @property @@ -183,10 +171,7 @@ class HTTPConnection(_HTTPConnection, object): def _prepare_conn(self, conn): self.sock = conn - # the _tunnel_host attribute was added in python 2.6.3 (via - # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do - # not have them. - if getattr(self, '_tunnel_host', None): + if self._tunnel_host: # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() # Mark this connection as not reusable @@ -217,13 +202,13 @@ class HTTPConnection(_HTTPConnection, object): self.endheaders() if body is not None: - stringish_types = six.string_types + (six.binary_type,) + stringish_types = six.string_types + (bytes,) if isinstance(body, stringish_types): body = (body,) for chunk in body: if not chunk: continue - if not isinstance(chunk, six.binary_type): + if not isinstance(chunk, bytes): chunk = chunk.encode('utf8') len_str = hex(len(chunk))[2:] self.send(len_str.encode('utf-8')) @@ -242,7 +227,7 @@ class HTTPSConnection(HTTPConnection): def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - ssl_context=None, **kw): + ssl_context=None, server_hostname=None, **kw): HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) @@ -250,6 +235,7 @@ class HTTPSConnection(HTTPConnection): self.key_file = key_file self.cert_file = cert_file self.ssl_context = ssl_context + self.server_hostname = server_hostname # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) @@ -270,6 +256,7 @@ class HTTPSConnection(HTTPConnection): keyfile=self.key_file, certfile=self.cert_file, ssl_context=self.ssl_context, + server_hostname=self.server_hostname ) @@ -312,12 +299,9 @@ class VerifiedHTTPSConnection(HTTPSConnection): def connect(self): # Add certificate verification conn = self._new_conn() - hostname = self.host - if getattr(self, '_tunnel_host', None): - # _tunnel_host was added in Python 2.6.3 - # (See: http://hg.python.org/cpython/rev/0f57b30a152f) + if self._tunnel_host: self.sock = conn # Calls self._set_hostport(), so self.host is # self._tunnel_host below. @@ -328,6 +312,10 @@ class VerifiedHTTPSConnection(HTTPSConnection): # Override the host with the one we're requesting data from. hostname = self._tunnel_host + server_hostname = hostname + if self.server_hostname is not None: + server_hostname = self.server_hostname + is_time_off = datetime.date.today() < RECENT_DATE if is_time_off: warnings.warn(( @@ -352,7 +340,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): certfile=self.cert_file, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, - server_hostname=hostname, + server_hostname=server_hostname, ssl_context=context) if self.assert_fingerprint: @@ -373,7 +361,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): 'for details.)'.format(hostname)), SubjectAltNameWarning ) - _match_hostname(cert, self.assert_hostname or hostname) + _match_hostname(cert, self.assert_hostname or server_hostname) self.is_verified = ( context.verify_mode == ssl.CERT_REQUIRED or diff --git a/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py b/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py index 8fcb0bce..f7a8f193 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py +++ b/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py @@ -89,7 +89,7 @@ class ConnectionPool(object): # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 -_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) +_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK} class HTTPConnectionPool(ConnectionPool, RequestMethods): @@ -313,7 +313,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Catch possible read timeouts thrown as SSL errors. If not the # case, rethrow the original. We need to do this because of: # http://bugs.python.org/issue10272 - if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 + if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python < 2.7.4 raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) def _make_request(self, conn, method, url, timeout=_Default, chunked=False, @@ -375,7 +375,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): try: try: # Python 2.7, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) - except TypeError: # Python 2.6 and older, Python 3 + except TypeError: # Python 3 try: httplib_response = conn.getresponse() except Exception as e: @@ -801,17 +801,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): Establish tunnel connection early, because otherwise httplib would improperly set Host: header to proxy's IP:port. """ - # Python 2.7+ - try: - set_tunnel = conn.set_tunnel - except AttributeError: # Platform-specific: Python 2.6 - set_tunnel = conn._set_tunnel - - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older - set_tunnel(self._proxy_host, self.port) - else: - set_tunnel(self._proxy_host, self.port, self.proxy_headers) - + conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) conn.connect() def _new_conn(self): diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/_appengine_environ.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/_appengine_environ.py new file mode 100644 index 00000000..f3e00942 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/_appengine_environ.py @@ -0,0 +1,30 @@ +""" +This module provides means to detect the App Engine environment. +""" + +import os + + +def is_appengine(): + return (is_local_appengine() or + is_prod_appengine() or + is_prod_appengine_mvms()) + + +def is_appengine_sandbox(): + return is_appengine() and not is_prod_appengine_mvms() + + +def is_local_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Development/' in os.environ['SERVER_SOFTWARE']) + + +def is_prod_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and + not is_prod_appengine_mvms()) + + +def is_prod_appengine_mvms(): + return os.environ.get('GAE_VM', False) == 'true' diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py index 06586352..1c2332cb 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py @@ -39,8 +39,8 @@ urllib3 on Google App Engine: """ from __future__ import absolute_import +import io import logging -import os import warnings from ..packages.six.moves.urllib.parse import urljoin @@ -53,11 +53,11 @@ from ..exceptions import ( SSLError ) -from ..packages.six import BytesIO from ..request import RequestMethods from ..response import HTTPResponse from ..util.timeout import Timeout from ..util.retry import Retry +from . import _appengine_environ try: from google.appengine.api import urlfetch @@ -239,7 +239,7 @@ class AppEngineManager(RequestMethods): original_response = HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), msg=urlfetch_resp.header_msg, headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, @@ -247,7 +247,7 @@ class AppEngineManager(RequestMethods): ) return HTTPResponse( - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, original_response=original_response, @@ -280,26 +280,10 @@ class AppEngineManager(RequestMethods): return retries -def is_appengine(): - return (is_local_appengine() or - is_prod_appengine() or - is_prod_appengine_mvms()) +# Alias methods from _appengine_environ to maintain public API interface. - -def is_appengine_sandbox(): - return is_appengine() and not is_prod_appengine_mvms() - - -def is_local_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Development/' in os.environ['SERVER_SOFTWARE']) - - -def is_prod_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and - not is_prod_appengine_mvms()) - - -def is_prod_appengine_mvms(): - return os.environ.get('GAE_VM', False) == 'true' +is_appengine = _appengine_environ.is_appengine +is_appengine_sandbox = _appengine_environ.is_appengine_sandbox +is_local_appengine = _appengine_environ.is_local_appengine +is_prod_appengine = _appengine_environ.is_prod_appengine +is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py index 642e99ed..8ea127c5 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py @@ -43,8 +43,7 @@ class NTLMConnectionPool(HTTPSConnectionPool): log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', self.num_connections, self.host, self.authurl) - headers = {} - headers['Connection'] = 'Keep-Alive' + headers = {'Connection': 'Keep-Alive'} req_header = 'Authorization' resp_header = 'www-authenticate' diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py index 7787d4e4..f5bc7d83 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py @@ -163,6 +163,9 @@ def _dnsname_to_stdlib(name): from ASCII bytes. We need to idna-encode that string to get it back, and then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). + + If the name cannot be idna-encoded then we return None signalling that + the name given should be skipped. """ def idna_encode(name): """ @@ -172,14 +175,19 @@ def _dnsname_to_stdlib(name): """ from pipenv.patched.notpip._vendor import idna - for prefix in [u'*.', u'.']: - if name.startswith(prefix): - name = name[len(prefix):] - return prefix.encode('ascii') + idna.encode(name) - return idna.encode(name) + try: + for prefix in [u'*.', u'.']: + if name.startswith(prefix): + name = name[len(prefix):] + return prefix.encode('ascii') + idna.encode(name) + return idna.encode(name) + except idna.core.IDNAError: + return None name = idna_encode(name) - if sys.version_info >= (3, 0): + if name is None: + return None + elif sys.version_info >= (3, 0): name = name.decode('utf-8') return name @@ -223,9 +231,10 @@ def get_subj_alt_name(peer_cert): # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8 # decoded. This is pretty frustrating, but that's what the standard library # does with certificates, and so we need to attempt to do the same. + # We also want to skip over names which cannot be idna encoded. names = [ - ('DNS', _dnsname_to_stdlib(name)) - for name in ext.get_values_for_type(x509.DNSName) + ('DNS', name) for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) + if name is not None ] names.extend( ('IP Address', str(name)) diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py b/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py index 75b80dcf..740db377 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py +++ b/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py @@ -16,7 +16,7 @@ def backport_makefile(self, mode="r", buffering=None, encoding=None, """ Backport of ``socket.makefile`` from Python 3.5. """ - if not set(mode) <= set(["r", "w", "b"]): + if not set(mode) <= {"r", "w", "b"}: raise ValueError( "invalid mode %r (only r, w, b allowed)" % (mode,) ) diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/ordered_dict.py b/pipenv/patched/notpip/_vendor/urllib3/packages/ordered_dict.py deleted file mode 100644 index 4479363c..00000000 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/ordered_dict.py +++ /dev/null @@ -1,259 +0,0 @@ -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. -# Copyright 2009 Raymond Hettinger, released under the MIT License. -# http://code.activestate.com/recipes/576693/ -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py b/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py index 62a177a7..b48752f8 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py +++ b/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py @@ -9,8 +9,7 @@ import sys # ipaddress has been backported to 2.6+ in pypi. If it is installed on the # system, use it to handle IPAddress ServerAltnames (this was added in # python-3.5) otherwise only do DNS matching. This allows -# backports.ssl_match_hostname to continue to be used all the way back to -# python-2.4. +# backports.ssl_match_hostname to continue to be used in Python 2.7. try: from pipenv.patched.notpip._vendor import ipaddress except ImportError: diff --git a/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py b/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py index 506a3c9b..fe5491cf 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py +++ b/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py @@ -47,6 +47,7 @@ _key_fields = ( 'key__socks_options', # dict 'key_assert_hostname', # bool or string 'key_assert_fingerprint', # str + 'key_server_hostname', #str ) #: The namedtuple class used to construct keys for the connection pool. diff --git a/pipenv/patched/notpip/_vendor/urllib3/request.py b/pipenv/patched/notpip/_vendor/urllib3/request.py index 1be33341..8f2f44bb 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/request.py +++ b/pipenv/patched/notpip/_vendor/urllib3/request.py @@ -36,7 +36,7 @@ class RequestMethods(object): explicitly. """ - _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + _encode_url_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS'} def __init__(self, headers=None): self.headers = headers or {} diff --git a/pipenv/patched/notpip/_vendor/urllib3/response.py b/pipenv/patched/notpip/_vendor/urllib3/response.py index 9873cb94..c112690b 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/response.py +++ b/pipenv/patched/notpip/_vendor/urllib3/response.py @@ -11,7 +11,7 @@ from .exceptions import ( BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked, IncompleteRead, InvalidHeader ) -from .packages.six import string_types as basestring, binary_type, PY3 +from .packages.six import string_types as basestring, PY3 from .packages.six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head @@ -23,7 +23,7 @@ class DeflateDecoder(object): def __init__(self): self._first_try = True - self._data = binary_type() + self._data = b'' self._obj = zlib.decompressobj() def __getattr__(self, name): @@ -69,9 +69,9 @@ class GzipDecoder(object): return getattr(self._obj, name) def decompress(self, data): - ret = binary_type() + ret = bytearray() if self._state == GzipDecoderState.SWALLOW_DATA or not data: - return ret + return bytes(ret) while True: try: ret += self._obj.decompress(data) @@ -81,16 +81,40 @@ class GzipDecoder(object): self._state = GzipDecoderState.SWALLOW_DATA if previous_state == GzipDecoderState.OTHER_MEMBERS: # Allow trailing garbage acceptable in other gzip clients - return ret + return bytes(ret) raise data = self._obj.unused_data if not data: - return ret + return bytes(ret) self._state = GzipDecoderState.OTHER_MEMBERS self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) +class MultiDecoder(object): + """ + From RFC7231: + If one or more encodings have been applied to a representation, the + sender that applied the encodings MUST generate a Content-Encoding + header field that lists the content codings in the order in which + they were applied. + """ + + def __init__(self, modes): + self._decoders = [_get_decoder(m.strip()) for m in modes.split(',')] + + def flush(self): + return self._decoders[0].flush() + + def decompress(self, data): + for d in reversed(self._decoders): + data = d.decompress(data) + return data + + def _get_decoder(mode): + if ',' in mode: + return MultiDecoder(mode) + if mode == 'gzip': return GzipDecoder() @@ -159,7 +183,7 @@ class HTTPResponse(io.IOBase): self.msg = msg self._request_url = request_url - if body and isinstance(body, (basestring, binary_type)): + if body and isinstance(body, (basestring, bytes)): self._body = body self._pool = pool @@ -283,8 +307,13 @@ class HTTPResponse(io.IOBase): # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None and content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + if self._decoder is None: + if content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + elif ',' in content_encoding: + encodings = [e.strip() for e in content_encoding.split(',') if e.strip() in self.CONTENT_DECODERS] + if len(encodings): + self._decoder = _get_decoder(content_encoding) def _decode(self, data, decode_content, flush_decoder): """ diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/connection.py b/pipenv/patched/notpip/_vendor/urllib3/util/connection.py index 5cf488f4..5ad70b2f 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/connection.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/connection.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import socket from .wait import NoWayToWaitForSocketError, wait_for_read +from ..contrib import _appengine_environ def is_connection_dropped(conn): # Platform-specific @@ -105,6 +106,13 @@ def _has_ipv6(host): sock = None has_ipv6 = False + # App Engine doesn't support IPV6 sockets and actually has a quota on the + # number of sockets that can be used, so just early out here instead of + # creating a socket needlessly. + # See https://github.com/urllib3/urllib3/issues/1446 + if _appengine_environ.is_appengine_sandbox(): + return False + if socket.has_ipv6: # has_ipv6 returns true if cPython was compiled with IPv6 support. # It does not tell us if the system has IPv6 support enabled. To diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/response.py b/pipenv/patched/notpip/_vendor/urllib3/util/response.py index 67cf730a..3d548648 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/response.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/response.py @@ -59,8 +59,14 @@ def assert_header_parsing(headers): get_payload = getattr(headers, 'get_payload', None) unparsed_data = None - if get_payload: # Platform-specific: Python 3. - unparsed_data = get_payload() + if get_payload: + # get_payload is actually email.message.Message.get_payload; + # we're only interested in the result if it's not a multipart message + if not headers.is_multipart(): + payload = get_payload() + + if isinstance(payload, (bytes, str)): + unparsed_data = payload if defects or unparsed_data: raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data) diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/retry.py b/pipenv/patched/notpip/_vendor/urllib3/util/retry.py index 7ad3dc66..e7d0abd6 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/retry.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/retry.py @@ -115,7 +115,7 @@ class Retry(object): (most errors are resolved immediately by a second try without a delay). urllib3 will sleep for:: - {backoff factor} * (2 ^ ({number of total retries} - 1)) + {backoff factor} * (2 ** ({number of total retries} - 1)) seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py b/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py index a0868c22..b16d6523 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py @@ -56,9 +56,8 @@ except ImportError: OP_NO_COMPRESSION = 0x20000 -# Python 2.7 and earlier didn't have inet_pton on non-Linux -# so we fallback on inet_aton in those cases. This means that -# we can only detect IPv4 addresses in this case. +# Python 2.7 doesn't have inet_pton on non-Linux so we fallback on inet_aton in +# those cases. This means that we can only detect IPv4 addresses in this case. if hasattr(socket, 'inet_pton'): inet_pton = socket.inet_pton else: @@ -67,7 +66,7 @@ else: from pipenv.patched.notpip._vendor import ipaddress def inet_pton(_, host): - if isinstance(host, six.binary_type): + if isinstance(host, bytes): host = host.decode('ascii') return ipaddress.ip_address(host) @@ -115,10 +114,7 @@ try: except ImportError: import sys - class SSLContext(object): # Platform-specific: Python 2 & 3.1 - supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or - (3, 2) <= sys.version_info) - + class SSLContext(object): # Platform-specific: Python 2 def __init__(self, protocol_version): self.protocol = protocol_version # Use default values from a real SSLContext @@ -141,12 +137,6 @@ except ImportError: raise SSLError("CA directories not supported in older Pythons") def set_ciphers(self, cipher_suite): - if not self.supports_set_ciphers: - raise TypeError( - 'Your version of Python does not support setting ' - 'a custom cipher suite. Please upgrade to Python ' - '2.7, 3.2, or later if you need this functionality.' - ) self.ciphers = cipher_suite def wrap_socket(self, socket, server_hostname=None, server_side=False): @@ -167,10 +157,7 @@ except ImportError: 'ssl_version': self.protocol, 'server_side': server_side, } - if self.supports_set_ciphers: # Platform-specific: Python 2.7+ - return wrap_socket(socket, ciphers=self.ciphers, **kwargs) - else: # Platform-specific: Python 2.6 - return wrap_socket(socket, **kwargs) + return wrap_socket(socket, ciphers=self.ciphers, **kwargs) def assert_fingerprint(cert, fingerprint): @@ -276,6 +263,8 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, """ context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + context.set_ciphers(ciphers or DEFAULT_CIPHERS) + # Setting the default here, as we may have no ssl module on import cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs @@ -291,9 +280,6 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, context.options |= options - if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 - context.set_ciphers(ciphers or DEFAULT_CIPHERS) - context.verify_mode = cert_reqs if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 # We do our own verification, including fingerprints and alternative @@ -316,8 +302,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, A pre-made :class:`SSLContext` object. If none is provided, one will be created using :func:`create_urllib3_context`. :param ciphers: - A string of ciphers we wish the client to support. This is not - supported on Python 2.6 as the ssl module does not support it. + A string of ciphers we wish the client to support. :param ca_cert_dir: A directory containing CA certificates in multiple separate files, as supported by OpenSSL's -CApath flag or the capath argument to @@ -334,7 +319,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if ca_certs or ca_cert_dir: try: context.load_verify_locations(ca_certs, ca_cert_dir) - except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + except IOError as e: # Platform-specific: Python 2.7 raise SSLError(e) # Py33 raises FileNotFoundError which subclasses OSError # These are not equivalent unless we check the errno attribute @@ -378,7 +363,7 @@ def is_ipaddress(hostname): :param str hostname: Hostname to examine. :return: True if the hostname is an IP address, False otherwise. """ - if six.PY3 and isinstance(hostname, six.binary_type): + if six.PY3 and isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. hostname = hostname.decode('ascii') diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/wait.py b/pipenv/patched/notpip/_vendor/urllib3/util/wait.py index fa686eff..4db71baf 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/wait.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/wait.py @@ -43,9 +43,6 @@ if sys.version_info >= (3, 5): else: # Old and broken Pythons. def _retry_on_intr(fn, timeout): - if timeout is not None and timeout <= 0: - return fn(timeout) - if timeout is None: deadline = float("inf") else: @@ -117,7 +114,7 @@ def _have_working_poll(): # from libraries like eventlet/greenlet. try: poll_obj = select.poll() - poll_obj.poll(0) + _retry_on_intr(poll_obj.poll, 0) except (AttributeError, OSError): return False else: diff --git a/pipenv/patched/notpip/appdirs.LICENSE.txt b/pipenv/patched/notpip/appdirs.LICENSE.txt new file mode 100644 index 00000000..107c6140 --- /dev/null +++ b/pipenv/patched/notpip/appdirs.LICENSE.txt @@ -0,0 +1,23 @@ +# This is the MIT license + +Copyright (c) 2010 ActiveState Software Inc. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/pipenv/patched/notpip/distro.LICENSE b/pipenv/patched/notpip/distro.LICENSE new file mode 100644 index 00000000..e06d2081 --- /dev/null +++ b/pipenv/patched/notpip/distro.LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/pipenv/patched/notpip/ipaddress.LICENSE b/pipenv/patched/notpip/ipaddress.LICENSE new file mode 100644 index 00000000..41bd16ba --- /dev/null +++ b/pipenv/patched/notpip/ipaddress.LICENSE @@ -0,0 +1,50 @@ +This package is a modified version of cpython's ipaddress module. +It is therefore distributed under the PSF license, as follows: + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are +retained in Python alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. diff --git a/pipenv/patched/notpip/packaging.LICENSE.APACHE b/pipenv/patched/notpip/packaging.LICENSE.APACHE new file mode 100644 index 00000000..4947287f --- /dev/null +++ b/pipenv/patched/notpip/packaging.LICENSE.APACHE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/pipenv/patched/notpip/pyparsing.LICENSE b/pipenv/patched/notpip/pyparsing.LICENSE new file mode 100644 index 00000000..1bf98523 --- /dev/null +++ b/pipenv/patched/notpip/pyparsing.LICENSE @@ -0,0 +1,18 @@ +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pipenv/patched/notpip/retrying.LICENSE b/pipenv/patched/notpip/retrying.LICENSE new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/pipenv/patched/notpip/retrying.LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/pipenv/patched/notpip/six.LICENSE b/pipenv/patched/notpip/six.LICENSE new file mode 100644 index 00000000..365d1074 --- /dev/null +++ b/pipenv/patched/notpip/six.LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2010-2018 Benjamin Peterson + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pipenv/patched/notpip/urllib3.LICENSE b/pipenv/patched/notpip/urllib3.LICENSE new file mode 100644 index 00000000..1c3283ee --- /dev/null +++ b/pipenv/patched/notpip/urllib3.LICENSE @@ -0,0 +1,19 @@ +This is the MIT license: http://www.opensource.org/licenses/mit-license.php + +Copyright 2008-2016 Andrey Petrov and contributors (see CONTRIBUTORS.txt) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/pipenv/patched/notpip/webencodings.LICENSE b/pipenv/patched/notpip/webencodings.LICENSE new file mode 100644 index 00000000..3d0d3e70 --- /dev/null +++ b/pipenv/patched/notpip/webencodings.LICENSE @@ -0,0 +1,31 @@ +Copyright (c) 2012 by Simon Sapin. + +Some rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 1ebebcd7bb4bb8915eb2e4e684b2174f4f36fe69 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 20 Mar 2019 00:04:23 -0400 Subject: [PATCH 03/22] Update piptools Signed-off-by: Dan Ryan --- .../vendoring/patches/patched/piptools.patch | 427 ++++++------------ 1 file changed, 150 insertions(+), 277 deletions(-) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 3799ccf4..e0693cc1 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -1,33 +1,28 @@ diff --git a/pipenv/patched/piptools/_compat/__init__.py b/pipenv/patched/piptools/_compat/__init__.py -index 1fa3805..c0ecec8 100644 +index e4ac717..19adcbc 100644 --- a/pipenv/patched/piptools/_compat/__init__.py +++ b/pipenv/patched/piptools/_compat/__init__.py -@@ -27,6 +27,8 @@ from .pip_compat import ( - cmdoptions, - get_installed_distributions, - PyPI, -+ SafeFileCache, -+ InstallationError, - install_req_from_line, +@@ -31,4 +31,6 @@ from .pip_compat import ( install_req_from_editable, + stdlib_pkgs, + DEV_PKGS, ++ SafeFileCache, ++ InstallationError ) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py -index 28da51f..c466ef0 100644 +index 82ccb8b..44de54b 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py -@@ -1,45 +1,55 @@ +@@ -1,48 +1,51 @@ # -*- coding=utf-8 -*- -import importlib - --import pip --import pkg_resources +__all__ = [ + "InstallRequirement", + "parse_requirements", + "RequirementSet", -+ "user_cache_dir", + "FAVORITE_HASH", + "is_file_url", ++ "path_to_url", + "url_to_path", + "PackageFinder", + "FormatControl", @@ -36,15 +31,20 @@ index 28da51f..c466ef0 100644 + "cmdoptions", + "get_installed_distributions", + "PyPI", -+ "SafeFileCache", -+ "InstallationError", -+ "parse_version", -+ "pip_version", -+ "install_req_from_editable", ++ "stdlib_pkgs", ++ "DEV_PKGS", + "install_req_from_line", -+ "user_cache_dir" ++ "install_req_from_editable", ++ "user_cache_dir", ++ "SafeFileCache", ++ "InstallationError" +] +-import pip +-import pkg_resources ++import os ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") + -def do_import(module_path, subimport=None, old_path=None): - old_path = old_path or module_path - prefixes = ["pip._internal", "pip"] @@ -68,6 +68,7 @@ index 28da51f..c466ef0 100644 -user_cache_dir = do_import('utils.appdirs', 'user_cache_dir') -FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH') -is_file_url = do_import('download', 'is_file_url') +-path_to_url = do_import('download', 'path_to_url') -url_to_path = do_import('download', 'url_to_path') -PackageFinder = do_import('index', 'PackageFinder') -FormatControl = do_import('index', 'FormatControl') @@ -76,13 +77,23 @@ index 28da51f..c466ef0 100644 -cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions') -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') -PyPI = do_import('models.index', 'PyPI') -+from pipenv.vendor.appdirs import user_cache_dir +-stdlib_pkgs = do_import('utils.compat', 'stdlib_pkgs', old_path='compat') +-DEV_PKGS = do_import('commands.freeze', 'DEV_PKGS') +- +-# pip 18.1 has refactored InstallRequirement constructors use by pip-tools. +-if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('18.1'): +- install_req_from_line = InstallRequirement.from_line +- install_req_from_editable = InstallRequirement.from_editable +-else: +- install_req_from_line = do_import('req.constructors', 'install_req_from_line') +- install_req_from_editable = do_import('req.constructors', 'install_req_from_editable') +from pip_shims.shims import ( + InstallRequirement, + parse_requirements, + RequirementSet, + FAVORITE_HASH, + is_file_url, ++ path_to_url, + url_to_path, + PackageFinder, + FormatControl, @@ -91,50 +102,59 @@ index 28da51f..c466ef0 100644 + cmdoptions, + get_installed_distributions, + PyPI, ++ stdlib_pkgs, ++ DEV_PKGS, ++ install_req_from_line, ++ install_req_from_editable, ++ USER_CACHE_DIR as user_cache_dir, + SafeFileCache, -+ InstallationError, -+ parse_version, -+ pip_version, ++ InstallationError +) +diff --git a/pipenv/patched/piptools/locations.py b/pipenv/patched/piptools/locations.py +index 4e6174c..9e0c6f1 100644 +--- a/pipenv/patched/piptools/locations.py ++++ b/pipenv/patched/piptools/locations.py +@@ -5,7 +5,11 @@ from .click import secho + from ._compat import user_cache_dir - # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. --if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('18.1'): -+if parse_version(pip_version) < parse_version('18.1'): - install_req_from_line = InstallRequirement.from_line - install_req_from_editable = InstallRequirement.from_editable - else: -- install_req_from_line = do_import('req.constructors', 'install_req_from_line') -- install_req_from_editable = do_import('req.constructors', 'install_req_from_editable') -+ from pip_shims.shims import ( -+ install_req_from_editable, install_req_from_line -+ ) -diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py -index 08dabe1..480ad1e 100644 ---- a/pipenv/patched/piptools/repositories/local.py -+++ b/pipenv/patched/piptools/repositories/local.py -@@ -56,7 +56,7 @@ class LocalRequirementsRepository(BaseRepository): - if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin): - project, version, _ = as_tuple(existing_pin) - return make_install_requirement( -- project, version, ireq.extras, constraint=ireq.constraint -+ project, version, ireq.extras, constraint=ireq.constraint, markers=ireq.markers - ) - else: - return self.repository.find_best_match(ireq, prereleases) + # The user_cache_dir helper comes straight from pip itself +-CACHE_DIR = user_cache_dir('pip-tools') ++try: ++ from pipenv.environments import PIPENV_CACHE_DIR ++ CACHE_DIR = PIPENV_CACHE_DIR ++except ImportError: ++ CACHE_DIR = user_cache_dir('pipenv') + + # NOTE + # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index bf69803..31b85b9 100644 +index e54ae08..6f42749 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -1,7 +1,7 @@ - # coding: utf-8 +@@ -2,6 +2,7 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -- + +import copy import hashlib import os from contextlib import contextmanager -@@ -15,13 +15,22 @@ from .._compat import ( +@@ -10,6 +11,14 @@ from shutil import rmtree + import pip + import pkg_resources + ++from packaging.requirements import Requirement ++from packaging.specifiers import SpecifierSet, Specifier ++ ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") ++from pip_shims.shims import VcsSupport, WheelCache, InstallationError ++from pip_shims.shims import Resolver as PipResolver ++ ++ + from .._compat import ( + is_file_url, + url_to_path, +@@ -18,13 +27,15 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -143,25 +163,17 @@ index bf69803..31b85b9 100644 + InstallRequirement, + SafeFileCache ) -+os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") -+from pip_shims.shims import do_import, VcsSupport, WheelCache -+from packaging.requirements import Requirement -+from packaging.specifiers import SpecifierSet, Specifier -+InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) -+from pip._internal.resolve import Resolver as PipResolver -+ --from ..cache import CACHE_DIR -+from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR + from ..cache import CACHE_DIR from ..exceptions import NoCandidateFound -from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, -+ make_install_requirement, clean_requires_python) ++ make_install_requirement, clean_requires_python) from .base import BaseRepository try: -@@ -31,10 +40,44 @@ except ImportError: +@@ -34,10 +45,44 @@ except ImportError: def RequirementTracker(): yield @@ -210,18 +222,20 @@ index bf69803..31b85b9 100644 class PyPIRepository(BaseRepository): -@@ -46,8 +89,9 @@ class PyPIRepository(BaseRepository): +@@ -49,10 +94,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ -- def __init__(self, pip_options, session): -+ def __init__(self, pip_options, session, use_json=False): +- def __init__(self, pip_options, session, build_isolation=False): ++ def __init__(self, pip_options, session, build_isolation=False, use_json=False): self.session = session -+ self.use_json = use_json self.pip_options = pip_options + self.build_isolation = build_isolation ++ self.use_json = use_json index_urls = [pip_options.index_url] + pip_options.extra_index_urls -@@ -73,6 +117,10 @@ class PyPIRepository(BaseRepository): + if pip_options.no_index: +@@ -82,6 +128,10 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -232,7 +246,7 @@ index bf69803..31b85b9 100644 # Setup file paths self.freshen_build_caches() -@@ -113,10 +161,13 @@ class PyPIRepository(BaseRepository): +@@ -122,10 +172,13 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -240,22 +254,36 @@ index bf69803..31b85b9 100644 + all_candidates = clean_requires_python(self.find_all_candidates(ireq.name)) candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) - matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), +- prereleases=prereleases) + try: + matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), - prereleases=prereleases) ++ prereleases=prereleases) + except TypeError: + matching_versions = [candidate.version for candidate in all_candidates] # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,25 +177,87 @@ class PyPIRepository(BaseRepository): +@@ -135,9 +188,65 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( - best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint -- ) -+ best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint -+ ) ++ best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint + ) + ++ def get_dependencies(self, ireq): ++ json_results = set() ++ ++ if self.use_json: ++ try: ++ json_results = self.get_json_dependencies(ireq) ++ except TypeError: ++ json_results = set() ++ ++ legacy_results = self.get_legacy_dependencies(ireq) ++ json_results.update(legacy_results) ++ ++ return json_results + + def get_json_dependencies(self, ireq): + @@ -294,47 +322,16 @@ index bf69803..31b85b9 100644 + try: + if ireq not in self._json_dep_cache: + self._json_dep_cache[ireq] = [g for g in gen(ireq)] - -- def resolve_reqs(self, download_dir, ireq, wheel_cache): ++ + return set(self._json_dep_cache[ireq]) + except Exception: + return set() + -+ def get_dependencies(self, ireq): -+ json_results = set() -+ -+ if self.use_json: -+ try: -+ json_results = self.get_json_dependencies(ireq) -+ except TypeError: -+ json_results = set() -+ -+ legacy_results = self.get_legacy_dependencies(ireq) -+ json_results.update(legacy_results) -+ -+ return json_results -+ -+ def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist=None): + def resolve_reqs(self, download_dir, ireq, wheel_cache): results = None -+ setup_requires = {} -+ dist = None -+ ireq.isolated = False -+ ireq._wheel_cache = wheel_cache -+ try: - from pip._internal.operations.prepare import RequirementPreparer -- from pip._internal.resolve import Resolver as PipResolver - except ImportError: -- # Pip 9 and below -+ # Pip 9 and below - reqset = RequirementSet( - self.build_dir, - self.source_dir, - download_dir=download_dir, - wheel_download_dir=self._wheel_download_dir, +@@ -153,7 +262,7 @@ class PyPIRepository(BaseRepository): session=self.session, -+ ignore_installed=True, -+ ignore_compatibility=False, wheel_cache=wheel_cache ) - results = reqset._prepare_file(self.finder, ireq) @@ -342,147 +339,38 @@ index bf69803..31b85b9 100644 else: # pip >= 10 preparer_kwargs = { -@@ -159,13 +272,14 @@ class PyPIRepository(BaseRepository): - 'finder': self.finder, - 'session': self.session, +@@ -170,8 +279,9 @@ class PyPIRepository(BaseRepository): 'upgrade_strategy': "to-satisfy-only", -- 'force_reinstall': False, -+ 'force_reinstall': True, + 'force_reinstall': False, 'ignore_dependencies': False, - 'ignore_requires_python': False, + 'ignore_requires_python': True, 'ignore_installed': True, ++ 'ignore_compatibility': False, 'isolated': False, 'wheel_cache': wheel_cache, -- 'use_user_site': False -+ 'use_user_site': False, -+ 'ignore_compatibility': False - } - resolver = None - preparer = None -@@ -177,15 +291,109 @@ class PyPIRepository(BaseRepository): - resolver_kwargs['preparer'] = preparer - reqset = RequirementSet() - ireq.is_direct = True -- reqset.add_requirement(ireq) -+ # reqset.add_requirement(ireq) + 'use_user_site': False +@@ -190,11 +300,17 @@ class PyPIRepository(BaseRepository): resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() - -- return set(results) ++ + cleanup_fn = getattr(reqset, "cleanup_files", None) + if cleanup_fn is not None: + try: + cleanup_fn() + except OSError: + pass -+ -+ if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): -+ if ireq.editable: -+ self._source_dir = TemporaryDirectory(fs_str("source")) -+ ireq.ensure_has_source_dir(self.source_dir) -+ -+ if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): -+ # Collect setup_requires info from local eggs. -+ # Do this after we call the preparer on these reqs to make sure their -+ # egg info has been created -+ from pipenv.utils import chdir -+ with chdir(ireq.setup_py_dir): -+ try: -+ from setuptools.dist import distutils -+ dist = distutils.core.run_setup(ireq.setup_py) -+ except InstallationError: -+ ireq.run_egg_info() -+ except (TypeError, ValueError, AttributeError): -+ pass -+ if not dist: -+ try: -+ dist = ireq.get_dist() -+ except (ImportError, ValueError, TypeError, AttributeError): -+ pass -+ if ireq.editable and dist: -+ setup_requires = getattr(dist, "extras_require", None) -+ if not setup_requires: -+ setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)} -+ if not getattr(ireq, 'req', None): -+ try: -+ ireq.req = dist.as_requirement() if dist else None -+ except (ValueError, TypeError) as e: -+ pass + + return set(results) - def get_dependencies(self, ireq): -+ # Convert setup_requires dict into a somewhat usable form. -+ if setup_requires: -+ for section in setup_requires: -+ python_version = section -+ not_python = not (section.startswith('[') and ':' in section) -+ -+ # This is for cleaning up :extras: formatted markers -+ # by adding them to the results of the resolver -+ # since any such extra would have been returned as a result anyway -+ for value in setup_requires[section]: -+ # This is a marker. -+ if value.startswith('[') and ':' in value: -+ python_version = value[1:-1] -+ not_python = False -+ # Strip out other extras. -+ if value.startswith('[') and ':' not in value: -+ not_python = True -+ -+ if ':' not in value: -+ try: -+ if not not_python: -+ results.add(InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))) -+ # Anything could go wrong here -- can't be too careful. -+ except Exception: -+ pass -+ -+ # this section properly creates 'python_version' markers for cross-python -+ # virtualenv creation and for multi-python compatibility. -+ requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else resolver.requires_python -+ if requires_python: -+ marker_str = '' -+ # This corrects a logic error from the previous code which said that if -+ # we Encountered any 'requires_python' attributes, basically only create a -+ # single result no matter how many we resolved. This should fix -+ # a majority of the remaining non-deterministic resolution issues. -+ if any(requires_python.startswith(op) for op in Specifier._operators.keys()): -+ # We are checking first if we have leading specifier operator -+ # if not, we can assume we should be doing a == comparison -+ specifierset = SpecifierSet(requires_python) -+ # for multiple specifiers, the correct way to represent that in -+ # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')` -+ from passa.internals.specifiers import cleanup_pyspecs -+ marker_str = str(Marker(" and ".join(dedup([ -+ "python_version {0[0]} '{0[1]}'".format(spec) -+ for spec in cleanup_pyspecs(specifierset) -+ ])))) -+ # The best way to add markers to a requirement is to make a separate requirement -+ # with only markers on it, and then to transfer the object istelf -+ marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker -+ if ireq in results: -+ results.remove(ireq) -+ print(marker_to_add) -+ ireq.req.marker = marker_to_add -+ -+ results = set(results) if results else set() -+ return results, ireq -+ + def get_legacy_dependencies(self, ireq): """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -200,6 +408,7 @@ class PyPIRepository(BaseRepository): - # If a download_dir is passed, pip will unnecessarely - # archive the entire source directory - download_dir = None -+ - elif ireq.link and not ireq.link.is_artifact: - # No download_dir for VCS sources. This also works around pip - # using git-checkout-index, which gets rid of the .git dir. -@@ -214,7 +423,8 @@ class PyPIRepository(BaseRepository): +@@ -223,7 +339,8 @@ class PyPIRepository(BaseRepository): wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) prev_tracker = os.environ.get('PIP_REQ_TRACKER') try: @@ -492,7 +380,7 @@ index bf69803..31b85b9 100644 finally: if 'PIP_REQ_TRACKER' in os.environ: if prev_tracker: -@@ -236,6 +446,10 @@ class PyPIRepository(BaseRepository): +@@ -245,6 +362,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -503,7 +391,7 @@ index bf69803..31b85b9 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -243,24 +457,22 @@ class PyPIRepository(BaseRepository): +@@ -252,24 +373,16 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. @@ -512,22 +400,16 @@ index bf69803..31b85b9 100644 - matching_versions = list( - ireq.specifier.filter((candidate.version for candidate in all_candidates))) - matching_candidates = candidates_by_version[matching_versions[0]] -+ ### Modification -- this is much more efficient.... -+ ### modification again -- still more efficient + matching_candidates = ( + c for c in clean_requires_python(self.find_all_candidates(ireq.name)) + if c.version in ireq.specifier + ) -+ # candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) -+ # matching_versions = list( -+ # ireq.specifier.filter((candidate.version for candidate in all_candidates))) -+ # matching_candidates = candidates_by_version[matching_versions[0]] return { - self._get_file_hash(candidate.location) - for candidate in matching_candidates + h for h in map(lambda c: self._hash_cache.get_hash(c.location), -+ matching_candidates) if h is not None ++ matching_candidates) if h is not None } - def _get_file_hash(self, location): @@ -541,26 +423,17 @@ index bf69803..31b85b9 100644 def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index c2d323c..d5a471d 100644 +index 494d385..b642bc9 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py -@@ -13,7 +13,7 @@ from . import click - from .cache import DependencyCache - from .exceptions import UnsupportedConstraint - from .logging import log --from .utils import (format_requirement, format_specifier, full_groupby, -+from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, - is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) - - green = partial(click.style, fg='green') -@@ -27,6 +27,7 @@ class RequirementSummary(object): - def __init__(self, ireq): +@@ -28,6 +28,7 @@ class RequirementSummary(object): self.req = ireq.req self.key = key_from_req(ireq.req) -+ self.markers = ireq.markers self.extras = str(sorted(ireq.extras)) ++ self.markers = ireq.markers self.specifier = str(ireq.specifier) + def __eq__(self, other): @@ -119,7 +120,7 @@ class Resolver(object): @staticmethod def check_constraints(constraints): @@ -570,7 +443,7 @@ index c2d323c..d5a471d 100644 msg = ('pip-compile does not support URLs as packages, unless they are editable. ' 'Perhaps add -e option?') raise UnsupportedConstraint(msg, constraint) -@@ -155,6 +156,13 @@ class Resolver(object): +@@ -155,6 +156,12 @@ class Resolver(object): # NOTE we may be losing some info on dropped reqs here combined_ireq.req.specifier &= ireq.req.specifier combined_ireq.constraint &= ireq.constraint @@ -580,11 +453,10 @@ index c2d323c..d5a471d 100644 + _markers = combined_ireq.markers._markers + if not isinstance(_markers[0], (tuple, list)): + combined_ireq.markers._markers = [_markers, 'and', ireq.markers._markers] -+ # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq -@@ -272,6 +280,15 @@ class Resolver(object): +@@ -272,6 +279,15 @@ class Resolver(object): for dependency in self.repository.get_dependencies(ireq): yield dependency return @@ -600,7 +472,7 @@ index c2d323c..d5a471d 100644 elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) -@@ -282,7 +299,7 @@ class Resolver(object): +@@ -282,7 +298,7 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) @@ -610,29 +482,32 @@ index c2d323c..d5a471d 100644 # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4'] dependency_strings = self.dependency_cache[ireq] diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py -index 2360a04..6f62eb9 100644 +index 9b4b4c2..8875543 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py -@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, +@@ -2,10 +2,17 @@ + from __future__ import (absolute_import, division, print_function, + unicode_literals) - import os ++import os import sys -+import six from itertools import chain, groupby from collections import OrderedDict - from contextlib import contextmanager -@@ -11,11 +12,78 @@ from contextlib import contextmanager + ++import six ++ ++from pipenv.vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier ++from pipenv.vendor.packaging.version import Version, InvalidVersion, parse as parse_version ++from pipenv.vendor.packaging.markers import Marker, Op, Value, Variable ++ from ._compat import install_req_from_line from .click import style -+from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier -+from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -+from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable - - +@@ -14,6 +21,71 @@ from .click import style UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} ++ +def simplify_markers(ireq): + """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`" + @@ -700,7 +575,7 @@ index 2360a04..6f62eb9 100644 def key_from_ireq(ireq): """Get a standardized key for an InstallRequirement.""" if ireq.req is None and ireq.link is not None: -@@ -41,30 +109,61 @@ def comment(text): +@@ -39,16 +111,51 @@ def comment(text): return style(text, fg='green') @@ -723,13 +598,10 @@ index 2360a04..6f62eb9 100644 + return install_req_from_line( + str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), + constraint=constraint) - - --def format_requirement(ireq, marker=None): ++ ++ +def _requirement_to_str_lowercase_name(requirement): - """ -- Generic formatter for pretty printing InstallRequirements to the terminal -- in a less verbose way than using its `__str__` method. ++ """ + Formats a packaging.requirements.Requirement with a lowercase name. + + This is simply a copy of @@ -740,7 +612,7 @@ index 2360a04..6f62eb9 100644 + lowercasing the entire result, which would lowercase the name, *and* other, + important stuff that should not be lowercased (such as the marker). See + this issue for more information: https://github.com/pypa/pipenv/issues/2113. - """ ++ """ + parts = [requirement.name.lower()] + + if requirement.extras: @@ -756,9 +628,10 @@ index 2360a04..6f62eb9 100644 + parts.append("; {0}".format(requirement.marker)) + + return "".join(parts) -+ -+ -+def format_requirement(ireq, marker=None): + + + def format_requirement(ireq, marker=None, hashes=None): +@@ -59,10 +166,10 @@ def format_requirement(ireq, marker=None, hashes=None): if ireq.editable: line = '-e {}'.format(ireq.link) else: @@ -770,5 +643,5 @@ index 2360a04..6f62eb9 100644 + if marker and ';' not in line: + line = '{}; {}'.format(line, marker) - return line - + if hashes: + for hash_ in sorted(hashes): From 00e8685be97ce529e90cef36ac2a29238b40773c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 20 Mar 2019 00:19:19 -0400 Subject: [PATCH 04/22] update pip shims Signed-off-by: Dan Ryan --- pipenv/vendor/pip_shims/__init__.py | 2 +- pipenv/vendor/pip_shims/shims.py | 142 ++++++++++++++++++---------- 2 files changed, 91 insertions(+), 53 deletions(-) diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py index 22bf130f..32716bea 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import sys -__version__ = '0.3.2' +__version__ = '0.3.3' from . import shims diff --git a/pipenv/vendor/pip_shims/shims.py b/pipenv/vendor/pip_shims/shims.py index 34de6906..5ea2691f 100644 --- a/pipenv/vendor/pip_shims/shims.py +++ b/pipenv/vendor/pip_shims/shims.py @@ -1,26 +1,38 @@ # -*- coding=utf-8 -*- -from collections import namedtuple -from contextlib import contextmanager import importlib import os import sys - +from collections import namedtuple +from contextlib import contextmanager import six -six.add_move(six.MovedAttribute("Callable", "collections", "collections.abc")) -from six.moves import Callable + +# format: off +six.add_move(six.MovedAttribute("Callable", "collections", "collections.abc")) # noqa +from six.moves import Callable # type: ignore # noqa # isort:skip + +# format: on class _shims(object): - CURRENT_PIP_VERSION = "18.1" + CURRENT_PIP_VERSION = "19.0.3" BASE_IMPORT_PATH = os.environ.get("PIP_SHIMS_BASE_MODULE", "pip") path_info = namedtuple("PathInfo", "path start_version end_version") def __dir__(self): result = list(self._locations.keys()) + list(self.__dict__.keys()) - result.extend(('__file__', '__doc__', '__all__', - '__docformat__', '__name__', '__path__', - '__package__', '__version__')) + result.extend( + ( + "__file__", + "__doc__", + "__all__", + "__docformat__", + "__name__", + "__path__", + "__package__", + "__version__", + ) + ) return result @classmethod @@ -34,13 +46,14 @@ class _shims(object): def __init__(self): # from .utils import _parse, get_package, STRING_TYPES from . import utils + self.utils = utils self._parse = utils._parse self.get_package = utils.get_package self.STRING_TYPES = utils.STRING_TYPES self._modules = { "pip": importlib.import_module(self.BASE_IMPORT_PATH), - "pip_shims.utils": utils + "pip_shims.utils": utils, } self.pip_version = getattr(self._modules["pip"], "__version__") version_types = ["post", "pre", "dev", "rc"] @@ -62,15 +75,15 @@ class _shims(object): ), "cmdoptions": ( ("cli.cmdoptions", "18.1", "9999"), - ("cmdoptions", "7.0.0", "18.0") + ("cmdoptions", "7.0.0", "18.0"), ), "Command": ( ("cli.base_command.Command", "18.1", "9999"), - ("basecommand.Command", "7.0.0", "18.0") + ("basecommand.Command", "7.0.0", "18.0"), ), "ConfigOptionParser": ( ("cli.parser.ConfigOptionParser", "18.1", "9999"), - ("baseparser.ConfigOptionParser", "7.0.0", "18.0") + ("baseparser.ConfigOptionParser", "7.0.0", "18.0"), ), "DistributionNotFound": ("exceptions.DistributionNotFound", "7.0.0", "9999"), "FAVORITE_HASH": ("utils.hashes.FAVORITE_HASH", "7.0.0", "9999"), @@ -80,55 +93,70 @@ class _shims(object): ), "FrozenRequirement": ( ("FrozenRequirement", "7.0.0", "9.0.3"), - ("operations.freeze.FrozenRequirement", "10.0.0", "9999") + ("operations.freeze.FrozenRequirement", "10.0.0", "9999"), ), "get_installed_distributions": ( ("utils.misc.get_installed_distributions", "10", "9999"), - ("utils.get_installed_distributions", "7", "9.0.3") + ("utils.get_installed_distributions", "7", "9.0.3"), ), "index_group": ( ("cli.cmdoptions.index_group", "18.1", "9999"), - ("cmdoptions.index_group", "7.0.0", "18.0") + ("cmdoptions.index_group", "7.0.0", "18.0"), ), "InstallRequirement": ("req.req_install.InstallRequirement", "7.0.0", "9999"), "InstallationError": ("exceptions.InstallationError", "7.0.0", "9999"), "UninstallationError": ("exceptions.UninstallationError", "7.0.0", "9999"), "DistributionNotFound": ("exceptions.DistributionNotFound", "7.0.0", "9999"), - "RequirementsFileParseError": ("exceptions.RequirementsFileParseError", "7.0.0", "9999"), - "BestVersionAlreadyInstalled": ("exceptions.BestVersionAlreadyInstalled", "7.0.0", "9999"), + "RequirementsFileParseError": ( + "exceptions.RequirementsFileParseError", + "7.0.0", + "9999", + ), + "BestVersionAlreadyInstalled": ( + "exceptions.BestVersionAlreadyInstalled", + "7.0.0", + "9999", + ), "BadCommand": ("exceptions.BadCommand", "7.0.0", "9999"), "CommandError": ("exceptions.CommandError", "7.0.0", "9999"), - "PreviousBuildDirError": ("exceptions.PreviousBuildDirError", "7.0.0", "9999"), + "PreviousBuildDirError": ( + "exceptions.PreviousBuildDirError", + "7.0.0", + "9999", + ), "install_req_from_editable": ( ("req.constructors.install_req_from_editable", "18.1", "9999"), - ("req.req_install.InstallRequirement.from_editable", "7.0.0", "18.0") + ("req.req_install.InstallRequirement.from_editable", "7.0.0", "18.0"), ), "install_req_from_line": ( ("req.constructors.install_req_from_line", "18.1", "9999"), - ("req.req_install.InstallRequirement.from_line", "7.0.0", "18.0") + ("req.req_install.InstallRequirement.from_line", "7.0.0", "18.0"), ), "is_archive_file": ("download.is_archive_file", "7.0.0", "9999"), "is_file_url": ("download.is_file_url", "7.0.0", "9999"), "unpack_url": ("download.unpack_url", "7.0.0", "9999"), "is_installable_dir": ( ("utils.misc.is_installable_dir", "10.0.0", "9999"), - ("utils.is_installable_dir", "7.0.0", "9.0.3") + ("utils.is_installable_dir", "7.0.0", "9.0.3"), ), "Link": ("index.Link", "7.0.0", "9999"), "make_abstract_dist": ( ("operations.prepare.make_abstract_dist", "10.0.0", "9999"), - ("req.req_set.make_abstract_dist", "7.0.0", "9.0.3") + ("req.req_set.make_abstract_dist", "7.0.0", "9.0.3"), ), "make_option_group": ( ("cli.cmdoptions.make_option_group", "18.1", "9999"), - ("cmdoptions.make_option_group", "7.0.0", "18.0") + ("cmdoptions.make_option_group", "7.0.0", "18.0"), ), "PackageFinder": ("index.PackageFinder", "7.0.0", "9999"), "parse_requirements": ("req.req_file.parse_requirements", "7.0.0", "9999"), - "parse_version": ("index.parse_version", "7.0.0", "9999"), "path_to_url": ("download.path_to_url", "7.0.0", "9999"), "PipError": ("exceptions.PipError", "7.0.0", "9999"), - "RequirementPreparer": ("operations.prepare.RequirementPreparer", "7", "9999"), + "RequirementPreparer": ( + "operations.prepare.RequirementPreparer", + "7", + "9999", + ), "RequirementSet": ("req.req_set.RequirementSet", "7.0.0", "9999"), "RequirementTracker": ("req.req_tracker.RequirementTracker", "7.0.0", "9999"), "Resolver": ("resolve.Resolver", "7.0.0", "9999"), @@ -140,10 +168,18 @@ class _shims(object): "Wheel": ("wheel.Wheel", "7.0.0", "9999"), "WheelCache": ( ("cache.WheelCache", "10.0.0", "9999"), - ("wheel.WheelCache", "7", "9.0.3") + ("wheel.WheelCache", "7", "9.0.3"), ), "WheelBuilder": ("wheel.WheelBuilder", "7.0.0", "9999"), "PyPI": ("models.index.PyPI", "7.0.0", "9999"), + "stdlib_pkgs": ( + ("utils.compat.stdlib_pkgs", "18.1", "9999"), + ("compat.stdlib_pkgs", "7", "18.0"), + ), + "DEV_PKGS": ( + ("commands.freeze.DEV_PKGS", "9.0.0", "9999"), + ({"setuptools", "pip", "distribute", "wheel"}, "7.0.0", "8.1.2"), + ), } def _ensure_methods(self, cls, classname, *methods): @@ -151,6 +187,7 @@ class _shims(object): if all(getattr(cls, m, None) for m in method_names): return cls new_functions = {} + class BaseFunc(Callable): def __init__(self, func_base, name, *args, **kwargs): self.func = func_base @@ -163,11 +200,7 @@ class _shims(object): new_functions[method_name] = classmethod(BaseFunc(fn, method_name)) if six.PY2: classname = classname.encode(sys.getdefaultencoding()) - type_ = type( - classname, - (cls,), - new_functions - ) + type_ = type(classname, (cls,), new_functions) return type_ def _get_module_paths(self, module, base_path=None): @@ -187,22 +220,23 @@ class _shims(object): new_to_old = {} for method_name, new_method_name in self._moves.get(original_target, {}).items(): module_paths = self._get_module_paths(new_method_name) - target = next(iter( - sorted(set([ - tgt for mod, tgt in map(self.get_package, module_paths) - ]))), None + target = next( + iter( + sorted(set([tgt for mod, tgt in map(self.get_package, module_paths)])) + ), + None, ) old_to_new[method_name] = { "target": target, "name": new_method_name, "location": self._locations[new_method_name], - "module": self._import(self._locations[new_method_name]) + "module": self._import(self._locations[new_method_name]), } new_to_old[new_method_name] = { "target": original_target, "name": method_name, "location": self._locations[original_target], - "module": original_import + "module": original_import, } return (old_to_new, new_to_old) @@ -218,9 +252,7 @@ class _shims(object): imported = self._modules[new_target] = new_to_old[new_name]["module"] method_map.append((old_method, remapped["module"])) if getattr(imported, "__class__", "") == type: - imported = self._ensure_methods( - imported, new_target, *method_map - ) + imported = self._ensure_methods(imported, new_target, *method_map) self._modules[new_target] = imported if imported: return imported @@ -231,8 +263,7 @@ class _shims(object): self.get_package(pth) for pth in self._get_module_paths(search_pth) ] moved_methods = [ - (base, target_cls) for base, target_cls - in module_paths if target_cls in moves + (base, target_cls) for base, target_cls in module_paths if target_cls in moves ] return next(iter(moved_methods), None) @@ -280,6 +311,8 @@ class _shims(object): def import_module(self, module): if module in self._modules: return self._modules[module] + if not isinstance(module, six.string_types): + return module try: imported = importlib.import_module(module) except ImportError: @@ -299,7 +332,8 @@ class _shims(object): for m, package_name in map(self.get_package, modules) ] imports = [ - getattr(m, pkg, self.none_or_ctxmanager(pkg)) for pkg, m in modules + getattr(m, pkg, self.none_or_ctxmanager(pkg)) + for pkg, m in modules if m is not None ] return next(iter(imports), None) @@ -327,15 +361,19 @@ class _shims(object): def get_pathinfo(self, module_path): assert isinstance(module_path, (list, tuple)) module_path, start_version, end_version = module_path - return self.path_info(module_path, self._parse(start_version), self._parse(end_version)) + return self.path_info( + module_path, self._parse(start_version), self._parse(end_version) + ) old_module = sys.modules[__name__] if __name__ in sys.modules else None module = sys.modules[__name__] = _shims() -module.__dict__.update({ - '__file__': __file__, - '__package__': __package__, - '__doc__': __doc__, - '__all__': module.__all__, - '__name__': __name__, -}) +module.__dict__.update( + { + "__file__": __file__, + "__package__": __package__, + "__doc__": __doc__, + "__all__": module.__all__, + "__name__": __name__, + } +) From fa31b64f3773971aa26732f15c1b7a83043935be Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 20 Mar 2019 12:15:00 -0400 Subject: [PATCH 05/22] Update vendored piptools and pip versions Signed-off-by: Dan Ryan --- pipenv/patched/notpip/_internal/resolve.py | 3 +- pipenv/patched/piptools/__main__.py | 2 +- pipenv/patched/piptools/_compat/__init__.py | 6 +- pipenv/patched/piptools/_compat/pip_compat.py | 38 ++-- pipenv/patched/piptools/locations.py | 6 +- pipenv/patched/piptools/logging.py | 9 +- pipenv/patched/piptools/pip.py | 30 +++ pipenv/patched/piptools/repositories/local.py | 3 +- pipenv/patched/piptools/repositories/pypi.py | 197 +++++------------- pipenv/patched/piptools/resolver.py | 5 +- pipenv/patched/piptools/scripts/compile.py | 67 +++--- pipenv/patched/piptools/sync.py | 46 ++-- pipenv/patched/piptools/utils.py | 55 +++-- pipenv/patched/piptools/writer.py | 37 ++-- tasks/vendoring/patches/patched/pip19.patch | 90 ++++---- .../vendoring/patches/patched/piptools.patch | 74 +++++-- 16 files changed, 325 insertions(+), 343 deletions(-) create mode 100644 pipenv/patched/piptools/pip.py diff --git a/pipenv/patched/notpip/_internal/resolve.py b/pipenv/patched/notpip/_internal/resolve.py index 79e4b409..36945de5 100644 --- a/pipenv/patched/notpip/_internal/resolve.py +++ b/pipenv/patched/notpip/_internal/resolve.py @@ -20,6 +20,7 @@ from pipenv.patched.notpip._internal.exceptions import ( ) from pipenv.patched.notpip._internal.req.constructors import install_req_from_req_string from pipenv.patched.notpip._internal.utils.logging import indent_log +from pipenv.patched.notpip._internal.req.req_install import InstallRequirement from pipenv.patched.notpip._internal.utils.misc import dist_in_usersite, ensure_dir from pipenv.patched.notpip._internal.utils.packaging import check_dist_requires_python from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -378,7 +379,7 @@ class Resolver(object): if hasattr(dist, '_DistInfoDistribution__dep_map'): for req in dist._DistInfoDistribution__dep_map[available]: req = InstallRequirement( - str(req), + req, req_to_install, isolated=self.isolated, wheel_cache=self.wheel_cache, diff --git a/pipenv/patched/piptools/__main__.py b/pipenv/patched/piptools/__main__.py index 22bd7879..b08b8494 100644 --- a/pipenv/patched/piptools/__main__.py +++ b/pipenv/patched/piptools/__main__.py @@ -12,5 +12,5 @@ cli.add_command(sync.cli, 'sync') # Enable ``python -m piptools ...``. -if __name__ == '__main__': # pragma: no cover +if __name__ == '__main__': # pragma: no branch cli() diff --git a/pipenv/patched/piptools/_compat/__init__.py b/pipenv/patched/piptools/_compat/__init__.py index c0ecec8a..19adcbc5 100644 --- a/pipenv/patched/piptools/_compat/__init__.py +++ b/pipenv/patched/piptools/_compat/__init__.py @@ -27,8 +27,10 @@ from .pip_compat import ( cmdoptions, get_installed_distributions, PyPI, - SafeFileCache, - InstallationError, install_req_from_line, install_req_from_editable, + stdlib_pkgs, + DEV_PKGS, + SafeFileCache, + InstallationError ) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py index c466ef04..715144a3 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -1,12 +1,11 @@ # -*- coding=utf-8 -*- - __all__ = [ "InstallRequirement", "parse_requirements", "RequirementSet", - "user_cache_dir", "FAVORITE_HASH", "is_file_url", + "path_to_url", "url_to_path", "PackageFinder", "FormatControl", @@ -15,22 +14,25 @@ __all__ = [ "cmdoptions", "get_installed_distributions", "PyPI", - "SafeFileCache", - "InstallationError", - "parse_version", - "pip_version", - "install_req_from_editable", + "stdlib_pkgs", + "DEV_PKGS", "install_req_from_line", - "user_cache_dir" + "install_req_from_editable", + "user_cache_dir", + "SafeFileCache", + "InstallationError" ] -from pipenv.vendor.appdirs import user_cache_dir +import os +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") + from pip_shims.shims import ( InstallRequirement, parse_requirements, RequirementSet, FAVORITE_HASH, is_file_url, + path_to_url, url_to_path, PackageFinder, FormatControl, @@ -39,17 +41,11 @@ from pip_shims.shims import ( cmdoptions, get_installed_distributions, PyPI, + stdlib_pkgs, + DEV_PKGS, + install_req_from_line, + install_req_from_editable, + USER_CACHE_DIR as user_cache_dir, SafeFileCache, - InstallationError, - parse_version, - pip_version, + InstallationError ) - -# pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if parse_version(pip_version) < parse_version('18.1'): - install_req_from_line = InstallRequirement.from_line - install_req_from_editable = InstallRequirement.from_editable -else: - from pip_shims.shims import ( - install_req_from_editable, install_req_from_line - ) diff --git a/pipenv/patched/piptools/locations.py b/pipenv/patched/piptools/locations.py index 0d460f64..9fcea0af 100644 --- a/pipenv/patched/piptools/locations.py +++ b/pipenv/patched/piptools/locations.py @@ -5,7 +5,11 @@ from .click import secho from ._compat import user_cache_dir # The user_cache_dir helper comes straight from pipenv.patched.notpip itself -CACHE_DIR = user_cache_dir('pip-tools') +try: + from pipenv.environments import PIPENV_CACHE_DIR + CACHE_DIR = PIPENV_CACHE_DIR +except ImportError: + CACHE_DIR = user_cache_dir('pipenv') # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/logging.py b/pipenv/patched/piptools/logging.py index 98f05287..f0bd1784 100644 --- a/pipenv/patched/piptools/logging.py +++ b/pipenv/patched/piptools/logging.py @@ -8,18 +8,19 @@ from . import click class LogContext(object): - def __init__(self, verbose=False): - self.verbose = verbose + def __init__(self, verbosity=0): + self.verbosity = verbosity def log(self, *args, **kwargs): click.secho(*args, **kwargs) def debug(self, *args, **kwargs): - if self.verbose: + if self.verbosity >= 1: self.log(*args, **kwargs) def info(self, *args, **kwargs): - self.log(*args, **kwargs) + if self.verbosity >= 0: + self.log(*args, **kwargs) def warning(self, *args, **kwargs): kwargs.setdefault('fg', 'yellow') diff --git a/pipenv/patched/piptools/pip.py b/pipenv/patched/piptools/pip.py new file mode 100644 index 00000000..0419a8ab --- /dev/null +++ b/pipenv/patched/piptools/pip.py @@ -0,0 +1,30 @@ +import optparse + +from ._compat import Command, cmdoptions + + +class PipCommand(Command): + name = 'PipCommand' + + +def get_pip_command(): + # Use pip's parser for pip.conf management and defaults. + # General options (find_links, index_url, extra_index_url, trusted_host, + # and pre) are defered to pip. + pip_command = PipCommand() + pip_command.parser.add_option(cmdoptions.no_binary()) + pip_command.parser.add_option(cmdoptions.only_binary()) + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, + pip_command.parser, + ) + pip_command.parser.insert_option_group(0, index_opts) + pip_command.parser.add_option(optparse.Option('--pre', action='store_true', default=False)) + + return pip_command + + +pip_command = get_pip_command() + +# Get default values of the pip's options (including options from pipenv.patched.notpip.conf). +pip_defaults = pip_command.parser.get_default_values() diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py index 480ad1ed..36bafdb9 100644 --- a/pipenv/patched/piptools/repositories/local.py +++ b/pipenv/patched/piptools/repositories/local.py @@ -56,7 +56,8 @@ class LocalRequirementsRepository(BaseRepository): if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin): project, version, _ = as_tuple(existing_pin) return make_install_requirement( - project, version, ireq.extras, constraint=ireq.constraint, markers=ireq.markers + project, version, ireq.extras, constraint=ireq.constraint, + markers=ireq.markers ) else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 2a0743a3..dff7cb88 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,12 +1,23 @@ # coding: utf-8 from __future__ import (absolute_import, division, print_function, unicode_literals) + import copy import hashlib import os from contextlib import contextmanager from shutil import rmtree +import pkg_resources + +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet, Specifier + +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") +from pip_shims.shims import VcsSupport, WheelCache, InstallationError, pip_version +from pip_shims.shims import Resolver as PipResolver + + from .._compat import ( is_file_url, url_to_path, @@ -19,18 +30,11 @@ from .._compat import ( InstallRequirement, SafeFileCache ) -os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") -from pip_shims.shims import do_import, VcsSupport, WheelCache -from packaging.requirements import Requirement -from packaging.specifiers import SpecifierSet, Specifier -InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) -from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver - -from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR +from ..cache import CACHE_DIR from ..exceptions import NoCandidateFound from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, - make_install_requirement, clean_requires_python) + make_install_requirement, clean_requires_python) from .base import BaseRepository try: @@ -89,23 +93,29 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ - def __init__(self, pip_options, session, use_json=False): + def __init__(self, pip_options, session, build_isolation=False, use_json=False): self.session = session - self.use_json = use_json self.pip_options = pip_options + self.build_isolation = build_isolation + self.use_json = use_json index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: index_urls = [] - self.finder = PackageFinder( - find_links=pip_options.find_links, - index_urls=index_urls, - trusted_hosts=pip_options.trusted_hosts, - allow_all_prereleases=pip_options.pre, - process_dependency_links=pip_options.process_dependency_links, - session=self.session, - ) + finder_kwargs = { + "find_links": pip_options.find_links, + "index_urls": index_urls, + "trusted_hosts": pip_options.trusted_hosts, + "allow_all_prereleases": pip_options.pre, + "session": self.session, + } + + # pip 19.0 has removed process_dependency_links from the PackageFinder constructor + if pkg_resources.parse_version(pip_version) < pkg_resources.parse_version('19.0'): + finder_kwargs["process_dependency_links"] = pip_options.process_dependency_links + + self.finder = PackageFinder(**finder_kwargs) # Caches # stores project_name => InstallationCandidate mappings for all @@ -165,7 +175,7 @@ class PyPIRepository(BaseRepository): candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) try: matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), - prereleases=prereleases) + prereleases=prereleases) except TypeError: matching_versions = [candidate.version for candidate in all_candidates] @@ -177,8 +187,22 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( - best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint - ) + best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint + ) + + def get_dependencies(self, ireq): + json_results = set() + + if self.use_json: + try: + json_results = self.get_json_dependencies(ireq) + except TypeError: + json_results = set() + + legacy_results = self.get_legacy_dependencies(ireq) + json_results.update(legacy_results) + + return json_results def get_json_dependencies(self, ireq): @@ -222,31 +246,13 @@ class PyPIRepository(BaseRepository): except Exception: return set() - def get_dependencies(self, ireq): - json_results = set() - - if self.use_json: - try: - json_results = self.get_json_dependencies(ireq) - except TypeError: - json_results = set() - - legacy_results = self.get_legacy_dependencies(ireq) - json_results.update(legacy_results) - - return json_results - - def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist=None): + def resolve_reqs(self, download_dir, ireq, wheel_cache): results = None - setup_requires = {} - dist = None - ireq.isolated = False - ireq._wheel_cache = wheel_cache - try: from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer + from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver except ImportError: - # Pip 9 and below + # Pip 9 and below reqset = RequirementSet( self.build_dir, self.source_dir, @@ -266,20 +272,20 @@ class PyPIRepository(BaseRepository): 'download_dir': download_dir, 'wheel_download_dir': self._wheel_download_dir, 'progress_bar': 'off', - 'build_isolation': False + 'build_isolation': self.build_isolation, } resolver_kwargs = { 'finder': self.finder, 'session': self.session, 'upgrade_strategy': "to-satisfy-only", - 'force_reinstall': True, + 'force_reinstall': False, 'ignore_dependencies': False, 'ignore_requires_python': True, 'ignore_installed': True, + 'ignore_compatibility': False, 'isolated': False, 'wheel_cache': wheel_cache, - 'use_user_site': False, - 'ignore_compatibility': False + 'use_user_site': False } resolver = None preparer = None @@ -302,94 +308,6 @@ class PyPIRepository(BaseRepository): cleanup_fn() except OSError: pass - - if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): - if ireq.editable: - self._source_dir = TemporaryDirectory(fs_str("source")) - ireq.ensure_has_source_dir(self.source_dir) - - if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): - # Collect setup_requires info from local eggs. - # Do this after we call the preparer on these reqs to make sure their - # egg info has been created - from pipenv.utils import chdir - with chdir(ireq.setup_py_dir): - try: - from setuptools.dist import distutils - dist = distutils.core.run_setup(ireq.setup_py) - except InstallationError: - ireq.run_egg_info() - except (TypeError, ValueError, AttributeError): - pass - if not dist: - try: - dist = ireq.get_dist() - except (ImportError, ValueError, TypeError, AttributeError): - pass - if ireq.editable and dist: - setup_requires = getattr(dist, "extras_require", None) - if not setup_requires: - setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)} - if not getattr(ireq, 'req', None): - try: - ireq.req = dist.as_requirement() if dist else None - except (ValueError, TypeError) as e: - pass - - # Convert setup_requires dict into a somewhat usable form. - if setup_requires: - for section in setup_requires: - python_version = section - not_python = not (section.startswith('[') and ':' in section) - - # This is for cleaning up :extras: formatted markers - # by adding them to the results of the resolver - # since any such extra would have been returned as a result anyway - for value in setup_requires[section]: - # This is a marker. - if value.startswith('[') and ':' in value: - python_version = value[1:-1] - not_python = False - # Strip out other extras. - if value.startswith('[') and ':' not in value: - not_python = True - - if ':' not in value: - try: - if not not_python: - results.add(InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))) - # Anything could go wrong here -- can't be too careful. - except Exception: - pass - - # this section properly creates 'python_version' markers for cross-python - # virtualenv creation and for multi-python compatibility. - requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else resolver.requires_python - if requires_python: - marker_str = '' - # This corrects a logic error from the previous code which said that if - # we Encountered any 'requires_python' attributes, basically only create a - # single result no matter how many we resolved. This should fix - # a majority of the remaining non-deterministic resolution issues. - if any(requires_python.startswith(op) for op in Specifier._operators.keys()): - # We are checking first if we have leading specifier operator - # if not, we can assume we should be doing a == comparison - specifierset = SpecifierSet(requires_python) - # for multiple specifiers, the correct way to represent that in - # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')` - from passa.internals.specifiers import cleanup_pyspecs - marker_str = str(Marker(" and ".join(dedup([ - "python_version {0[0]} '{0[1]}'".format(spec) - for spec in cleanup_pyspecs(specifierset) - ])))) - # The best way to add markers to a requirement is to make a separate requirement - # with only markers on it, and then to transfer the object istelf - marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker - if ireq in results: - results.remove(ireq) - print(marker_to_add) - ireq.req.marker = marker_to_add - results = set(results) if results else set() return results, ireq @@ -408,7 +326,6 @@ class PyPIRepository(BaseRepository): # If a download_dir is passed, pip will unnecessarely # archive the entire source directory download_dir = None - elif ireq.link and not ireq.link.is_artifact: # No download_dir for VCS sources. This also works around pip # using git-checkout-index, which gets rid of the .git dir. @@ -457,20 +374,14 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. - ### Modification -- this is much more efficient.... - ### modification again -- still more efficient matching_candidates = ( c for c in clean_requires_python(self.find_all_candidates(ireq.name)) if c.version in ireq.specifier ) - # candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) - # matching_versions = list( - # ireq.specifier.filter((candidate.version for candidate in all_candidates))) - # matching_candidates = candidates_by_version[matching_versions[0]] return { h for h in map(lambda c: self._hash_cache.get_hash(c.location), - matching_candidates) if h is not None + matching_candidates) if h is not None } @contextmanager diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 1f3e18c1..b642bc9c 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -13,7 +13,7 @@ from . import click from .cache import DependencyCache from .exceptions import UnsupportedConstraint from .logging import log -from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, +from .utils import (format_requirement, format_specifier, full_groupby, is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) green = partial(click.style, fg='green') @@ -27,8 +27,8 @@ class RequirementSummary(object): def __init__(self, ireq): self.req = ireq.req self.key = key_from_req(ireq.req) - self.markers = ireq.markers self.extras = str(sorted(ireq.extras)) + self.markers = ireq.markers self.specifier = str(ireq.specifier) def __eq__(self, other): @@ -162,7 +162,6 @@ class Resolver(object): _markers = combined_ireq.markers._markers if not isinstance(_markers[0], (tuple, list)): combined_ireq.markers._markers = [_markers, 'and', ireq.markers._markers] - # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq diff --git a/pipenv/patched/piptools/scripts/compile.py b/pipenv/patched/piptools/scripts/compile.py index 4625618a..2eaea9b3 100644 --- a/pipenv/patched/piptools/scripts/compile.py +++ b/pipenv/patched/piptools/scripts/compile.py @@ -2,7 +2,6 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -import optparse import os import sys import tempfile @@ -10,33 +9,30 @@ import tempfile from .._compat import ( install_req_from_line, parse_requirements, - cmdoptions, - Command, ) from .. import click from ..exceptions import PipToolsError from ..logging import log +from ..pip import get_pip_command, pip_defaults from ..repositories import LocalRequirementsRepository, PyPIRepository from ..resolver import Resolver from ..utils import (dedup, is_pinned_requirement, key_from_req, UNSAFE_PACKAGES) from ..writer import OutputWriter DEFAULT_REQUIREMENTS_FILE = 'requirements.in' - - -class PipCommand(Command): - name = 'PipCommand' +DEFAULT_REQUIREMENTS_OUTPUT_FILE = 'requirements.txt' @click.command() @click.version_option() -@click.option('-v', '--verbose', is_flag=True, help="Show more output") +@click.option('-v', '--verbose', count=True, help="Show more output") +@click.option('-q', '--quiet', count=True, help="Give less output") @click.option('-n', '--dry-run', is_flag=True, help="Only show what would happen, don't change anything") @click.option('-p', '--pre', is_flag=True, default=None, help="Allow resolving to prereleases (default is not)") @click.option('-r', '--rebuild', is_flag=True, help="Clear any caches upfront, rebuild from scratch") @click.option('-f', '--find-links', multiple=True, help="Look for archives in this directory or on this HTML page", envvar='PIP_FIND_LINKS') # noqa -@click.option('-i', '--index-url', help="Change index URL (defaults to PyPI)", envvar='PIP_INDEX_URL') +@click.option('-i', '--index-url', help="Change index URL (defaults to {})".format(pip_defaults.index_url), envvar='PIP_INDEX_URL') # noqa @click.option('--extra-index-url', multiple=True, help="Add additional index URL to search", envvar='PIP_EXTRA_INDEX_URL') # noqa @click.option('--cert', help="Path to alternate CA bundle.") @click.option('--client-cert', help="Path to SSL client certificate, a single file containing the private key and the certificate in PEM format.") # noqa @@ -65,27 +61,30 @@ class PipCommand(Command): @click.option('--max-rounds', default=10, help="Maximum number of rounds before resolving the requirements aborts.") @click.argument('src_files', nargs=-1, type=click.Path(exists=True, allow_dash=True)) -def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, +@click.option('--build-isolation/--no-build-isolation', is_flag=True, default=False, + help="Enable isolation when building a modern source distribution. " + "Build dependencies specified by PEP 518 must be already installed " + "if build isolation is disabled.") +def cli(verbose, quiet, dry_run, pre, rebuild, find_links, index_url, extra_index_url, cert, client_cert, trusted_host, header, index, emit_trusted_host, annotate, upgrade, upgrade_packages, output_file, allow_unsafe, generate_hashes, - src_files, max_rounds): + src_files, max_rounds, build_isolation): """Compiles requirements.txt from requirements.in specs.""" - log.verbose = verbose + log.verbosity = verbose - quiet if len(src_files) == 0: if os.path.exists(DEFAULT_REQUIREMENTS_FILE): src_files = (DEFAULT_REQUIREMENTS_FILE,) elif os.path.exists('setup.py'): src_files = ('setup.py',) - if not output_file: - output_file = 'requirements.txt' else: raise click.BadParameter(("If you do not specify an input file, " "the default is {} or setup.py").format(DEFAULT_REQUIREMENTS_FILE)) - if len(src_files) == 1 and src_files[0] == '-': - if not output_file: - raise click.BadParameter('--output-file is required if input is from stdin') + if src_files == ('-',) and not output_file: + raise click.BadParameter('--output-file is required if input is from stdin') + elif src_files == ('setup.py',) and not output_file: + output_file = DEFAULT_REQUIREMENTS_OUTPUT_FILE if len(src_files) > 1 and not output_file: raise click.BadParameter('--output-file is required if two or more input files are given.') @@ -127,17 +126,20 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, pip_options, _ = pip_command.parse_args(pip_args) session = pip_command._build_session(pip_options) - repository = PyPIRepository(pip_options, session) + repository = PyPIRepository(pip_options, session, build_isolation) + upgrade_install_reqs = {} # Proxy with a LocalRequirementsRepository if --upgrade is not specified # (= default invocation) if not upgrade and os.path.exists(dst_file): ireqs = parse_requirements(dst_file, finder=repository.finder, session=repository.session, options=pip_options) # Exclude packages from --upgrade-package/-P from the existing pins: We want to upgrade. - upgrade_pkgs_key = {key_from_req(install_req_from_line(pkg).req) for pkg in upgrade_packages} + upgrade_reqs_gen = (install_req_from_line(pkg) for pkg in upgrade_packages) + upgrade_install_reqs = {key_from_req(install_req.req): install_req for install_req in upgrade_reqs_gen} + existing_pins = {key_from_req(ireq.req): ireq for ireq in ireqs - if is_pinned_requirement(ireq) and key_from_req(ireq.req) not in upgrade_pkgs_key} + if is_pinned_requirement(ireq) and key_from_req(ireq.req) not in upgrade_install_reqs} repository = LocalRequirementsRepository(existing_pins, repository) log.debug('Using indexes:') @@ -178,6 +180,8 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, constraints.extend(parse_requirements( src_file, finder=repository.finder, session=repository.session, options=pip_options)) + constraints.extend(upgrade_install_reqs.values()) + # Filter out pip environment markers which do not match (PEP496) constraints = [req for req in constraints if req.markers is None or req.markers.evaluate()] @@ -236,32 +240,15 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, default_index_url=repository.DEFAULT_INDEX_URL, index_urls=repository.finder.index_urls, trusted_hosts=pip_options.trusted_hosts, - format_control=repository.finder.format_control) + format_control=repository.finder.format_control, + allow_unsafe=allow_unsafe) writer.write(results=results, unsafe_requirements=resolver.unsafe_constraints, reverse_dependencies=reverse_dependencies, primary_packages={key_from_req(ireq.req) for ireq in constraints if not ireq.constraint}, markers={key_from_req(ireq.req): ireq.markers for ireq in constraints if ireq.markers}, - hashes=hashes, - allow_unsafe=allow_unsafe) + hashes=hashes) if dry_run: log.warning('Dry-run, so nothing updated.') - - -def get_pip_command(): - # Use pip's parser for pip.conf management and defaults. - # General options (find_links, index_url, extra_index_url, trusted_host, - # and pre) are defered to pip. - pip_command = PipCommand() - pip_command.parser.add_option(cmdoptions.no_binary()) - pip_command.parser.add_option(cmdoptions.only_binary()) - index_opts = cmdoptions.make_option_group( - cmdoptions.index_group, - pip_command.parser, - ) - pip_command.parser.insert_option_group(0, index_opts) - pip_command.parser.add_option(optparse.Option('--pre', action='store_true', default=False)) - - return pip_command diff --git a/pipenv/patched/piptools/sync.py b/pipenv/patched/piptools/sync.py index 5c473916..f111764e 100644 --- a/pipenv/patched/piptools/sync.py +++ b/pipenv/patched/piptools/sync.py @@ -1,11 +1,13 @@ import collections import os import sys +import tempfile from subprocess import check_call +from piptools._compat import stdlib_pkgs, DEV_PKGS from . import click from .exceptions import IncompatibleRequirements, UnsupportedConstraint -from .utils import flat_map, format_requirement, key_from_ireq, key_from_req +from .utils import flat_map, format_requirement, key_from_ireq, key_from_req, get_hashes_from_ireq PACKAGES_TO_IGNORE = [ '-markerlib', @@ -13,9 +15,7 @@ PACKAGES_TO_IGNORE = [ 'pip-tools', 'pip-review', 'pkg-resources', - 'setuptools', - 'wheel', -] +] + list(stdlib_pkgs) + list(DEV_PKGS) def dependency_tree(installed_keys, root_key): @@ -120,33 +120,24 @@ def diff(compiled_requirements, installed_dists): return (to_install, to_uninstall) -def sync(to_install, to_uninstall, verbose=False, dry_run=False, pip_flags=None, install_flags=None): +def sync(to_install, to_uninstall, verbose=False, dry_run=False, install_flags=None): """ Install and uninstalls the given sets of modules. """ if not to_uninstall and not to_install: click.echo("Everything up-to-date") - if pip_flags is None: - pip_flags = [] - + pip_flags = [] if not verbose: pip_flags += ['-q'] - if os.environ.get('VIRTUAL_ENV'): - # find pip via PATH - pip = 'pip' - else: - # find pip in same directory as pip-sync entry-point script - pip = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), 'pip') - if to_uninstall: if dry_run: click.echo("Would uninstall:") for pkg in to_uninstall: click.echo(" {}".format(pkg)) else: - check_call([pip, 'uninstall', '-y'] + pip_flags + sorted(to_uninstall)) + check_call([sys.executable, '-m', 'pip', 'uninstall', '-y'] + pip_flags + sorted(to_uninstall)) if to_install: if install_flags is None: @@ -156,11 +147,22 @@ def sync(to_install, to_uninstall, verbose=False, dry_run=False, pip_flags=None, for ireq in to_install: click.echo(" {}".format(format_requirement(ireq))) else: - package_args = [] + # prepare requirement lines + req_lines = [] for ireq in sorted(to_install, key=key_from_ireq): - if ireq.editable: - package_args.extend(['-e', str(ireq.link or ireq.req)]) - else: - package_args.append(str(ireq.req)) - check_call([pip, 'install'] + pip_flags + install_flags + package_args) + ireq_hashes = get_hashes_from_ireq(ireq) + req_lines.append(format_requirement(ireq, hashes=ireq_hashes)) + + # save requirement lines to a temporary file + tmp_req_file = tempfile.NamedTemporaryFile(mode='wt', delete=False) + tmp_req_file.write('\n'.join(req_lines)) + tmp_req_file.close() + + try: + check_call( + [sys.executable, '-m', 'pip', 'install', '-r', tmp_req_file.name] + pip_flags + install_flags + ) + finally: + os.unlink(tmp_req_file.name) + return 0 diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index 6f62eb9d..88755431 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -4,22 +4,24 @@ from __future__ import (absolute_import, division, print_function, import os import sys -import six from itertools import chain, groupby from collections import OrderedDict -from contextlib import contextmanager + +import six + +from pipenv.vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier +from pipenv.vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from pipenv.vendor.packaging.markers import Marker, Op, Value, Variable from ._compat import install_req_from_line from .click import style -from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier -from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} + def simplify_markers(ireq): """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`" @@ -156,7 +158,11 @@ def _requirement_to_str_lowercase_name(requirement): return "".join(parts) -def format_requirement(ireq, marker=None): +def format_requirement(ireq, marker=None, hashes=None): + """ + Generic formatter for pretty printing InstallRequirements to the terminal + in a less verbose way than using its `__str__` method. + """ if ireq.editable: line = '-e {}'.format(ireq.link) else: @@ -165,6 +171,10 @@ def format_requirement(ireq, marker=None): if marker and ';' not in line: line = '{}; {}'.format(line, marker) + if hashes: + for hash_ in sorted(hashes): + line += " \\\n --hash={}".format(hash_) + return line @@ -252,6 +262,16 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): ... 'q': 'quux' ... } + For the values represented as lists, set use_lists=True: + + >>> assert lookup_table( + ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], + ... use_lists=True) == { + ... 'b': ['bar', 'baz'], + ... 'f': ['foo'], + ... 'q': ['qux', 'quux'] + ... } + The values of the resulting lookup table will be values, not sets. For extra power, you can even change the values while building up the LUT. @@ -336,15 +356,14 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() -# Borrowed from pew to avoid importing pew which imports psutil -# See https://github.com/berdario/pew/blob/master/pew/_utils.py#L82 -@contextmanager -def temp_environ(): - """Allow the ability to set os.environ temporarily""" - environ = dict(os.environ) - try: - yield - - finally: - os.environ.clear() - os.environ.update(environ) +def get_hashes_from_ireq(ireq): + """ + Given an InstallRequirement, return a list of string hashes in the format "{algorithm}:{hash}". + Return an empty list if there are no hashes in the requirement options. + """ + result = [] + ireq_hashes = ireq.options.get('hashes', {}) + for algorithm, hexdigests in ireq_hashes.items(): + for hash_ in hexdigests: + result.append("{}:{}".format(algorithm, hash_)) + return result diff --git a/pipenv/patched/piptools/writer.py b/pipenv/patched/piptools/writer.py index 97b6df94..9ac97792 100644 --- a/pipenv/patched/piptools/writer.py +++ b/pipenv/patched/piptools/writer.py @@ -1,8 +1,9 @@ import os +import sys from itertools import chain from ._compat import ExitStack -from .click import unstyle +from .click import unstyle, get_os_args from .io import AtomicSaver from .logging import log from .utils import comment, dedup, format_requirement, key_from_req, UNSAFE_PACKAGES @@ -11,7 +12,8 @@ from .utils import comment, dedup, format_requirement, key_from_req, UNSAFE_PACK class OutputWriter(object): def __init__(self, src_files, dst_file, dry_run, emit_header, emit_index, emit_trusted_host, annotate, generate_hashes, - default_index_url, index_urls, trusted_hosts, format_control): + default_index_url, index_urls, trusted_hosts, format_control, + allow_unsafe): self.src_files = src_files self.dst_file = dst_file self.dry_run = dry_run @@ -24,6 +26,7 @@ class OutputWriter(object): self.index_urls = index_urls self.trusted_hosts = trusted_hosts self.format_control = format_control + self.allow_unsafe = allow_unsafe def _sort_key(self, ireq): return (not ireq.editable, str(ireq.req).lower()) @@ -38,18 +41,9 @@ class OutputWriter(object): if custom_cmd: yield comment('# {}'.format(custom_cmd)) else: - params = [] - if not self.emit_index: - params += ['--no-index'] - if not self.emit_trusted_host: - params += ['--no-emit-trusted-host'] - if not self.annotate: - params += ['--no-annotate'] - if self.generate_hashes: - params += ["--generate-hashes"] - params += ['--output-file', self.dst_file] - params += self.src_files - yield comment('# pip-compile {}'.format(' '.join(params))) + prog = os.path.basename(sys.argv[0]) + args = ' '.join(get_os_args()) + yield comment('# {prog} {args}'.format(prog=prog, args=args)) yield comment('#') def write_index_options(self): @@ -82,7 +76,7 @@ class OutputWriter(object): yield '' def _iter_lines(self, results, unsafe_requirements, reverse_dependencies, - primary_packages, markers, hashes, allow_unsafe=False): + primary_packages, markers, hashes): for line in self.write_header(): yield line for line in self.write_flags(): @@ -110,32 +104,29 @@ class OutputWriter(object): primary_packages, marker=markers.get(key_from_req(ireq.req)), hashes=hashes) - if not allow_unsafe: + if not self.allow_unsafe: yield comment('# {}'.format(req)) else: yield req def write(self, results, unsafe_requirements, reverse_dependencies, - primary_packages, markers, hashes, allow_unsafe=False): + primary_packages, markers, hashes): with ExitStack() as stack: f = None if not self.dry_run: f = stack.enter_context(AtomicSaver(self.dst_file)) for line in self._iter_lines(results, unsafe_requirements, reverse_dependencies, - primary_packages, markers, hashes, allow_unsafe=allow_unsafe): + primary_packages, markers, hashes): log.info(line) if f: f.write(unstyle(line).encode('utf-8')) f.write(os.linesep.encode('utf-8')) def _format_requirement(self, ireq, reverse_dependencies, primary_packages, marker=None, hashes=None): - line = format_requirement(ireq, marker=marker) - ireq_hashes = (hashes if hashes is not None else {}).get(ireq) - if ireq_hashes: - for hash_ in sorted(ireq_hashes): - line += " \\\n --hash={}".format(hash_) + + line = format_requirement(ireq, marker=marker, hashes=ireq_hashes) if not self.annotate or key_from_req(ireq.req) in primary_packages: return line diff --git a/tasks/vendoring/patches/patched/pip19.patch b/tasks/vendoring/patches/patched/pip19.patch index dc45c445..1c10d5c5 100644 --- a/tasks/vendoring/patches/patched/pip19.patch +++ b/tasks/vendoring/patches/patched/pip19.patch @@ -1,7 +1,7 @@ -diff --git a/pipenv/patched/pip/_internal/download.py b/pipenv/patched/pip/_internal/download.py +diff --git a/pipenv/patched/notpip/_internal/download.py b/pipenv/patched/notpip/_internal/download.py index 2bbe1762..872af328 100644 ---- a/pipenv/patched/pip/_internal/download.py -+++ b/pipenv/patched/pip/_internal/download.py +--- a/pipenv/patched/notpip/_internal/download.py ++++ b/pipenv/patched/notpip/_internal/download.py @@ -77,7 +77,7 @@ def user_agent(): Return a string representing the user agent. """ @@ -11,10 +11,10 @@ index 2bbe1762..872af328 100644 "python": platform.python_version(), "implementation": { "name": platform.python_implementation(), -diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py +diff --git a/pipenv/patched/notpip/_internal/index.py b/pipenv/patched/notpip/_internal/index.py index 9eda3a35..67dd952c 100644 ---- a/pipenv/patched/pip/_internal/index.py -+++ b/pipenv/patched/pip/_internal/index.py +--- a/pipenv/patched/notpip/_internal/index.py ++++ b/pipenv/patched/notpip/_internal/index.py @@ -331,6 +331,9 @@ class PackageFinder(object): # The Session we'll use to make requests self.session = session @@ -156,10 +156,10 @@ index 9eda3a35..67dd952c 100644 def _find_name_version_sep(egg_info, canonical_name): -diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py +diff --git a/pipenv/patched/notpip/_internal/models/candidate.py b/pipenv/patched/notpip/_internal/models/candidate.py index 4475458a..6748957d 100644 ---- a/pipenv/patched/pip/_internal/models/candidate.py -+++ b/pipenv/patched/pip/_internal/models/candidate.py +--- a/pipenv/patched/notpip/_internal/models/candidate.py ++++ b/pipenv/patched/notpip/_internal/models/candidate.py @@ -13,11 +13,12 @@ class InstallationCandidate(KeyBasedCompareMixin): """Represents a potential "candidate" for installation. """ @@ -175,10 +175,10 @@ index 4475458a..6748957d 100644 super(InstallationCandidate, self).__init__( key=(self.project, self.version, self.location), -diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py +diff --git a/pipenv/patched/notpip/_internal/operations/prepare.py b/pipenv/patched/notpip/_internal/operations/prepare.py index 4f31dd5a..ed0c86b2 100644 ---- a/pipenv/patched/pip/_internal/operations/prepare.py -+++ b/pipenv/patched/pip/_internal/operations/prepare.py +--- a/pipenv/patched/notpip/_internal/operations/prepare.py ++++ b/pipenv/patched/notpip/_internal/operations/prepare.py @@ -17,7 +17,7 @@ from pip._internal.exceptions import ( from pip._internal.utils.compat import expanduser from pip._internal.utils.hashes import MissingHashes @@ -204,10 +204,10 @@ index 4f31dd5a..ed0c86b2 100644 req.populate_link(finder, upgrade_allowed, require_hashes) # We can't hit this spot and have populate_link return None. -diff --git a/pipenv/patched/pip/_internal/pep425tags.py b/pipenv/patched/pip/_internal/pep425tags.py +diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py index 1e782d1a..3c760ca3 100644 ---- a/pipenv/patched/pip/_internal/pep425tags.py -+++ b/pipenv/patched/pip/_internal/pep425tags.py +--- a/pipenv/patched/notpip/_internal/pep425tags.py ++++ b/pipenv/patched/notpip/_internal/pep425tags.py @@ -10,7 +10,10 @@ import sysconfig import warnings from collections import OrderedDict @@ -220,10 +220,10 @@ index 1e782d1a..3c760ca3 100644 from pip._internal.utils.compat import get_extension_suffixes from pip._internal.utils.typing import MYPY_CHECK_RUNNING -diff --git a/pipenv/patched/pip/_internal/req/req_install.py b/pipenv/patched/pip/_internal/req/req_install.py +diff --git a/pipenv/patched/notpip/_internal/req/req_install.py b/pipenv/patched/notpip/_internal/req/req_install.py index a4834b00..2c22e141 100644 ---- a/pipenv/patched/pip/_internal/req/req_install.py -+++ b/pipenv/patched/pip/_internal/req/req_install.py +--- a/pipenv/patched/notpip/_internal/req/req_install.py ++++ b/pipenv/patched/notpip/_internal/req/req_install.py @@ -588,7 +588,8 @@ class InstallRequirement(object): self.setup_py, self.link, ) @@ -256,10 +256,10 @@ index a4834b00..2c22e141 100644 install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ -diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py +diff --git a/pipenv/patched/notpip/_internal/req/req_set.py b/pipenv/patched/notpip/_internal/req/req_set.py index d1410e93..69a53bf2 100644 ---- a/pipenv/patched/pip/_internal/req/req_set.py -+++ b/pipenv/patched/pip/_internal/req/req_set.py +--- a/pipenv/patched/notpip/_internal/req/req_set.py ++++ b/pipenv/patched/notpip/_internal/req/req_set.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) class RequirementSet(object): @@ -288,11 +288,19 @@ index d1410e93..69a53bf2 100644 def cleanup_files(self): # type: () -> None -diff --git a/pipenv/patched/pip/_internal/resolve.py b/pipenv/patched/pip/_internal/resolve.py -index 33f572f1..fc8fcbe8 100644 ---- a/pipenv/patched/pip/_internal/resolve.py -+++ b/pipenv/patched/pip/_internal/resolve.py -@@ -58,7 +58,8 @@ class Resolver(object): +diff --git a/pipenv/patched/notpip/_internal/resolve.py b/pipenv/patched/notpip/_internal/resolve.py +index 33f572f1..aa827193 100644 +--- a/pipenv/patched/notpip/_internal/resolve.py ++++ b/pipenv/patched/notpip/_internal/resolve.py +@@ -19,6 +19,7 @@ from pip._internal.exceptions import ( + UnsupportedPythonVersion, + ) + from pip._internal.req.constructors import install_req_from_req_string ++from pip._internal.req.req_install import InstallRequirement + from pip._internal.utils.logging import indent_log + from pip._internal.utils.misc import dist_in_usersite, ensure_dir + from pip._internal.utils.packaging import check_dist_requires_python +@@ -58,7 +59,8 @@ class Resolver(object): force_reinstall, # type: bool isolated, # type: bool upgrade_strategy, # type: str @@ -302,7 +310,7 @@ index 33f572f1..fc8fcbe8 100644 ): # type: (...) -> None super(Resolver, self).__init__() -@@ -81,8 +82,12 @@ class Resolver(object): +@@ -81,8 +83,12 @@ class Resolver(object): self.ignore_dependencies = ignore_dependencies self.ignore_installed = ignore_installed self.ignore_requires_python = ignore_requires_python @@ -315,7 +323,7 @@ index 33f572f1..fc8fcbe8 100644 self._discovered_dependencies = \ defaultdict(list) # type: DefaultDict[str, List] -@@ -273,7 +278,8 @@ class Resolver(object): +@@ -273,7 +279,8 @@ class Resolver(object): def _resolve_one( self, requirement_set, # type: RequirementSet @@ -325,7 +333,7 @@ index 33f572f1..fc8fcbe8 100644 ): # type: (...) -> List[InstallRequirement] """Prepare a single requirements file. -@@ -298,11 +304,18 @@ class Resolver(object): +@@ -298,11 +305,18 @@ class Resolver(object): try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: @@ -345,7 +353,7 @@ index 33f572f1..fc8fcbe8 100644 more_reqs = [] # type: List[InstallRequirement] def add_req(subreq, extras_requested): -@@ -329,10 +342,14 @@ class Resolver(object): +@@ -329,10 +343,14 @@ class Resolver(object): # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): @@ -360,7 +368,7 @@ index 33f572f1..fc8fcbe8 100644 ) if not self.ignore_dependencies: -@@ -356,6 +373,20 @@ class Resolver(object): +@@ -356,6 +374,20 @@ class Resolver(object): for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) @@ -369,7 +377,7 @@ index 33f572f1..fc8fcbe8 100644 + if hasattr(dist, '_DistInfoDistribution__dep_map'): + for req in dist._DistInfoDistribution__dep_map[available]: + req = InstallRequirement( -+ str(req), ++ req, + req_to_install, + isolated=self.isolated, + wheel_cache=self.wheel_cache, @@ -381,10 +389,10 @@ index 33f572f1..fc8fcbe8 100644 if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took -diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py +diff --git a/pipenv/patched/notpip/_internal/utils/packaging.py b/pipenv/patched/notpip/_internal/utils/packaging.py index 7aaf7b5e..d56f0512 100644 ---- a/pipenv/patched/pip/_internal/utils/packaging.py -+++ b/pipenv/patched/pip/_internal/utils/packaging.py +--- a/pipenv/patched/notpip/_internal/utils/packaging.py ++++ b/pipenv/patched/notpip/_internal/utils/packaging.py @@ -37,7 +37,7 @@ def check_requires_python(requires_python): requires_python_specifier = specifiers.SpecifierSet(requires_python) @@ -407,10 +415,10 @@ index 7aaf7b5e..d56f0512 100644 try: if not check_requires_python(requires_python): raise exceptions.UnsupportedPythonVersion( -diff --git a/pipenv/patched/pip/_internal/utils/temp_dir.py b/pipenv/patched/pip/_internal/utils/temp_dir.py +diff --git a/pipenv/patched/notpip/_internal/utils/temp_dir.py b/pipenv/patched/notpip/_internal/utils/temp_dir.py index 2c81ad55..fbf0292a 100644 ---- a/pipenv/patched/pip/_internal/utils/temp_dir.py -+++ b/pipenv/patched/pip/_internal/utils/temp_dir.py +--- a/pipenv/patched/notpip/_internal/utils/temp_dir.py ++++ b/pipenv/patched/notpip/_internal/utils/temp_dir.py @@ -5,8 +5,10 @@ import itertools import logging import os.path @@ -477,10 +485,10 @@ index 2c81ad55..fbf0292a 100644 class AdjacentTempDirectory(TempDirectory): -diff --git a/pipenv/patched/pip/_internal/wheel.py b/pipenv/patched/pip/_internal/wheel.py +diff --git a/pipenv/patched/notpip/_internal/wheel.py b/pipenv/patched/notpip/_internal/wheel.py index 67bcc7f7..968cdff9 100644 ---- a/pipenv/patched/pip/_internal/wheel.py -+++ b/pipenv/patched/pip/_internal/wheel.py +--- a/pipenv/patched/notpip/_internal/wheel.py ++++ b/pipenv/patched/notpip/_internal/wheel.py @@ -114,7 +114,7 @@ def fix_script(path): firstline = script.readline() if not firstline.startswith(b'#!python'): diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index e0693cc1..87b5247c 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -10,7 +10,7 @@ index e4ac717..19adcbc 100644 + InstallationError ) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py -index 82ccb8b..44de54b 100644 +index 82ccb8b..715144a 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -1,48 +1,51 @@ @@ -43,7 +43,7 @@ index 82ccb8b..44de54b 100644 -import pip -import pkg_resources +import os -+os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") -def do_import(module_path, subimport=None, old_path=None): - old_path = old_path or module_path @@ -127,11 +127,25 @@ index 4e6174c..9e0c6f1 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the +diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py +index 08dabe1..36bafdb 100644 +--- a/pipenv/patched/piptools/repositories/local.py ++++ b/pipenv/patched/piptools/repositories/local.py +@@ -56,7 +56,8 @@ class LocalRequirementsRepository(BaseRepository): + if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin): + project, version, _ = as_tuple(existing_pin) + return make_install_requirement( +- project, version, ireq.extras, constraint=ireq.constraint ++ project, version, ireq.extras, constraint=ireq.constraint, ++ markers=ireq.markers + ) + else: + return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index e54ae08..6f42749 100644 +index e54ae08..4ff20e0 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -2,6 +2,7 @@ +@@ -2,14 +2,22 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) @@ -139,14 +153,15 @@ index e54ae08..6f42749 100644 import hashlib import os from contextlib import contextmanager -@@ -10,6 +11,14 @@ from shutil import rmtree - import pip + from shutil import rmtree + +-import pip import pkg_resources +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet, Specifier + -+os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") +from pip_shims.shims import VcsSupport, WheelCache, InstallationError +from pip_shims.shims import Resolver as PipResolver + @@ -154,7 +169,7 @@ index e54ae08..6f42749 100644 from .._compat import ( is_file_url, url_to_path, -@@ -18,13 +27,15 @@ from .._compat import ( +@@ -18,13 +26,15 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -173,7 +188,7 @@ index e54ae08..6f42749 100644 from .base import BaseRepository try: -@@ -34,10 +45,44 @@ except ImportError: +@@ -34,10 +44,44 @@ except ImportError: def RequirementTracker(): yield @@ -222,7 +237,7 @@ index e54ae08..6f42749 100644 class PyPIRepository(BaseRepository): -@@ -49,10 +94,11 @@ class PyPIRepository(BaseRepository): +@@ -49,10 +93,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -235,7 +250,16 @@ index e54ae08..6f42749 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -82,6 +128,10 @@ class PyPIRepository(BaseRepository): +@@ -67,7 +112,7 @@ class PyPIRepository(BaseRepository): + } + + # pip 19.0 has removed process_dependency_links from the PackageFinder constructor +- if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('19.0'): ++ if pkg_resources.parse_version(pip_shims.shims.pip_version) < pkg_resources.parse_version('19.0'): + finder_kwargs["process_dependency_links"] = pip_options.process_dependency_links + + self.finder = PackageFinder(**finder_kwargs) +@@ -82,6 +127,10 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -246,7 +270,7 @@ index e54ae08..6f42749 100644 # Setup file paths self.freshen_build_caches() -@@ -122,10 +172,13 @@ class PyPIRepository(BaseRepository): +@@ -122,10 +171,13 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -263,7 +287,7 @@ index e54ae08..6f42749 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -135,9 +188,65 @@ class PyPIRepository(BaseRepository): +@@ -135,9 +187,65 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( @@ -330,8 +354,12 @@ index e54ae08..6f42749 100644 def resolve_reqs(self, download_dir, ireq, wheel_cache): results = None try: -@@ -153,7 +262,7 @@ class PyPIRepository(BaseRepository): +@@ -151,9 +259,11 @@ class PyPIRepository(BaseRepository): + download_dir=download_dir, + wheel_download_dir=self._wheel_download_dir, session=self.session, ++ ignore_installed=True, ++ ignore_compatibility=False, wheel_cache=wheel_cache ) - results = reqset._prepare_file(self.finder, ireq) @@ -339,7 +367,7 @@ index e54ae08..6f42749 100644 else: # pip >= 10 preparer_kwargs = { -@@ -170,8 +279,9 @@ class PyPIRepository(BaseRepository): +@@ -170,8 +280,9 @@ class PyPIRepository(BaseRepository): 'upgrade_strategy': "to-satisfy-only", 'force_reinstall': False, 'ignore_dependencies': False, @@ -350,27 +378,29 @@ index e54ae08..6f42749 100644 'isolated': False, 'wheel_cache': wheel_cache, 'use_user_site': False -@@ -190,11 +300,17 @@ class PyPIRepository(BaseRepository): +@@ -190,11 +301,18 @@ class PyPIRepository(BaseRepository): resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() -+ + +- return set(results) + cleanup_fn = getattr(reqset, "cleanup_files", None) + if cleanup_fn is not None: + try: + cleanup_fn() + except OSError: + pass - - return set(results) ++ ++ results = set(results) if results else set() ++ return results, ireq - def get_dependencies(self, ireq): + def get_legacy_dependencies(self, ireq): """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -223,7 +339,8 @@ class PyPIRepository(BaseRepository): +@@ -223,7 +341,8 @@ class PyPIRepository(BaseRepository): wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) prev_tracker = os.environ.get('PIP_REQ_TRACKER') try: @@ -380,7 +410,7 @@ index e54ae08..6f42749 100644 finally: if 'PIP_REQ_TRACKER' in os.environ: if prev_tracker: -@@ -245,6 +362,10 @@ class PyPIRepository(BaseRepository): +@@ -245,6 +364,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -391,7 +421,7 @@ index e54ae08..6f42749 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -252,24 +373,16 @@ class PyPIRepository(BaseRepository): +@@ -252,24 +375,16 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. From 5e2ab186d7b8ef9e4295207e1e0221054e7cf858 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 20 Mar 2019 20:38:15 -0400 Subject: [PATCH 06/22] Update resolver to fix tempdir usage - Fix piptools patch - Fix windows ansi/colorama implementation - Write output even if no newline is found during resolution Signed-off-by: Dan Ryan --- pipenv/__init__.py | 5 ++- pipenv/patched/piptools/repositories/pypi.py | 7 ++-- pipenv/resolver.py | 6 +-- pipenv/utils.py | 9 ++--- .../vendoring/patches/patched/piptools.patch | 37 +++++++++++++++---- 5 files changed, 43 insertions(+), 21 deletions(-) diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 51aafed7..97625ddd 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -47,7 +47,10 @@ else: sys.stderr = get_wrapped_stream(stderr) sys.stdout = get_wrapped_stream(stdout) - +from .vendor.colorama import AnsiToWin32 +if os.name == "nt": + sys.stderr = AnsiToWin32(sys.stderr) + sys.stdout = AnsiToWin32(sys.stdout) from .cli import cli from . import resolver diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index dff7cb88..8dd369ac 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -248,9 +248,10 @@ class PyPIRepository(BaseRepository): def resolve_reqs(self, download_dir, ireq, wheel_cache): results = None + ireq.isolated = False + ireq._wheel_cache = wheel_cache try: from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer - from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver except ImportError: # Pip 9 and below reqset = RequirementSet( @@ -272,13 +273,13 @@ class PyPIRepository(BaseRepository): 'download_dir': download_dir, 'wheel_download_dir': self._wheel_download_dir, 'progress_bar': 'off', - 'build_isolation': self.build_isolation, + 'build_isolation': False, } resolver_kwargs = { 'finder': self.finder, 'session': self.session, 'upgrade_strategy': "to-satisfy-only", - 'force_reinstall': False, + 'force_reinstall': True, 'ignore_dependencies': False, 'ignore_requires_python': True, 'ignore_installed': True, diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 693c85ad..4f91350b 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -124,9 +124,6 @@ def resolve_packages(pre, clear, verbose, system, requirements_dir, packages): if "PIPENV_PYPI_MIRROR" in os.environ else None ) - # os.environ["PIP_NO_BUILD_ISOLATION"] = "1" - # os.environ["PIP_NO_USE_PEP517"] = "1" - # os.environ["PIP_NO_DEPS"] = "1" def resolve(packages, pre, project, sources, clear, system, requirements_dir=None): return resolve_deps( @@ -189,6 +186,9 @@ def main(): sys.stdout = get_wrapped_stream(stdout) from pipenv.vendor import colorama colorama.init() + if os.name == "nt": + sys.stderr = colorama.AnsiToWin32(sys.stderr) + sys.stdout = colorama.AnsiToWin32(sys.stdout) os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") os.environ["PYTHONIOENCODING"] = str("utf-8") parsed = handle_parsed_args(parsed) diff --git a/pipenv/utils.py b/pipenv/utils.py index 7b43c1f0..e615faca 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -840,18 +840,15 @@ def resolve(cmd, sp): result = c.expect(u"\n", timeout=environments.PIPENV_INSTALL_TIMEOUT) except (EOF, TIMEOUT): pass - if result is None: - break _out = c.subprocess.before if _out is not None: _out = decode_output("{0}".format(_out)) out += _out sp.text = to_native_string("{0}".format(_out[:100])) if environments.is_verbose(): - if _out is not None: - sp._hide_cursor() - sp.write(_out.rstrip()) - sp._show_cursor() + sp.hide_and_write(_out.rstrip()) + if result is None: + break c.block() if c.return_code != 0: sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 87b5247c..3d3cb99b 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -142,7 +142,7 @@ index 08dabe1..36bafdb 100644 else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index e54ae08..4ff20e0 100644 +index e54ae08..75b8208 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -2,14 +2,22 @@ @@ -287,7 +287,7 @@ index e54ae08..4ff20e0 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -135,9 +187,65 @@ class PyPIRepository(BaseRepository): +@@ -135,14 +187,71 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( @@ -353,8 +353,15 @@ index e54ae08..4ff20e0 100644 + def resolve_reqs(self, download_dir, ireq, wheel_cache): results = None ++ ireq.isolated = False ++ ireq._wheel_cache = wheel_cache try: -@@ -151,9 +259,11 @@ class PyPIRepository(BaseRepository): + from pip._internal.operations.prepare import RequirementPreparer +- from pip._internal.resolve import Resolver as PipResolver + except ImportError: + # Pip 9 and below + reqset = RequirementSet( +@@ -151,9 +260,11 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, @@ -367,7 +374,16 @@ index e54ae08..4ff20e0 100644 else: # pip >= 10 preparer_kwargs = { -@@ -170,8 +280,9 @@ class PyPIRepository(BaseRepository): +@@ -162,7 +273,7 @@ class PyPIRepository(BaseRepository): + 'download_dir': download_dir, + 'wheel_download_dir': self._wheel_download_dir, + 'progress_bar': 'off', +- 'build_isolation': self.build_isolation, ++ 'build_isolation': False, + } + resolver_kwargs = { + 'finder': self.finder, +@@ -170,8 +281,9 @@ class PyPIRepository(BaseRepository): 'upgrade_strategy': "to-satisfy-only", 'force_reinstall': False, 'ignore_dependencies': False, @@ -378,7 +394,12 @@ index e54ae08..4ff20e0 100644 'isolated': False, 'wheel_cache': wheel_cache, 'use_user_site': False -@@ -190,11 +301,18 @@ class PyPIRepository(BaseRepository): +@@ -186,15 +298,22 @@ class PyPIRepository(BaseRepository): + resolver_kwargs['preparer'] = preparer + reqset = RequirementSet() + ireq.is_direct = True +- reqset.add_requirement(ireq) ++ # reqset.add_requirement(ireq) resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) @@ -400,7 +421,7 @@ index e54ae08..4ff20e0 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -223,7 +341,8 @@ class PyPIRepository(BaseRepository): +@@ -223,7 +342,8 @@ class PyPIRepository(BaseRepository): wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) prev_tracker = os.environ.get('PIP_REQ_TRACKER') try: @@ -410,7 +431,7 @@ index e54ae08..4ff20e0 100644 finally: if 'PIP_REQ_TRACKER' in os.environ: if prev_tracker: -@@ -245,6 +364,10 @@ class PyPIRepository(BaseRepository): +@@ -245,6 +365,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -421,7 +442,7 @@ index e54ae08..4ff20e0 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -252,24 +375,16 @@ class PyPIRepository(BaseRepository): +@@ -252,24 +376,16 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. From 429a7bc2b712a6340e5d03ffe250357d9d7bb525 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 20 Mar 2019 21:45:39 -0400 Subject: [PATCH 07/22] Colorama wrap stream Signed-off-by: Dan Ryan --- pipenv/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 97625ddd..016f1012 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -49,8 +49,10 @@ sys.stderr = get_wrapped_stream(stderr) sys.stdout = get_wrapped_stream(stdout) from .vendor.colorama import AnsiToWin32 if os.name == "nt": - sys.stderr = AnsiToWin32(sys.stderr) - sys.stdout = AnsiToWin32(sys.stdout) + stderr_wrapper = AnsiToWin32(sys.stderr, autoreset=False, convert=None, strip=None) + stdout_wrapper = AnsiToWin32(sys.stdout, autoreset=False, convert=None, strip=None) + sys.stderr = stderr_wrapper.stream + sys.stdout = stdout_wrapper.stream from .cli import cli from . import resolver From 383020a720dcdd5d737217110a8eb4e30bd7c538 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 20 Mar 2019 22:24:13 -0400 Subject: [PATCH 08/22] Update test runners and fix resolver colorama calls Signed-off-by: Dan Ryan --- .azure-pipelines/steps/create-virtualenv.yml | 21 ++++++++++---------- .azure-pipelines/steps/run-tests.yml | 9 ++++++--- pipenv/resolver.py | 15 ++++++++++---- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.azure-pipelines/steps/create-virtualenv.yml b/.azure-pipelines/steps/create-virtualenv.yml index 3151d15e..4dbdb685 100644 --- a/.azure-pipelines/steps/create-virtualenv.yml +++ b/.azure-pipelines/steps/create-virtualenv.yml @@ -5,26 +5,27 @@ steps: if (!$env:PY_EXE) { $env:PY_EXE="python" } - Write-Host "##vso[task.setvariable variable=PY_EXE]"$env:PY_EXE + Write-Host "##vso[task.setvariable variable=PY_EXE]$env:PY_EXE" Write-Host "Found Python: $env:PY_EXE" - Invoke-Expression "$env:PY_EXE -m virtualenv D:\.venv" + Invoke-Expression "& '$env:PY_EXE' -m virtualenv D:\.venv" Write-Host "##vso[task.setvariable variable=VIRTUAL_ENV]D:\.venv" - Invoke-Expression "D:\.venv\Scripts\activate.ps1" + Invoke-Expression "& 'D:\.venv\Scripts\activate.ps1'" $env:VIRTUAL_ENV="D:\.venv" Write-Host "Installing local package..." - Invoke-Expression "$env:PY_EXE -m pip install -e .[test] --upgrade" + Invoke-Expression "& '$env:PY_EXE' -m pip install -e .[test] --upgrade" Write-Host "upgrading local package in virtual env" $venv_scripts = Join-Path -path D:\.venv -childpath Scripts $venv_py = Join-Path -path $venv_scripts -childpath python.exe - Invoke-Expression "$venv_py -m pip install -e .[test] --upgrade" + Write-Host "##vso[task.setvariable variable=VIRTUAL_ENV_PY]$venv_py" + Invoke-Expression "& '$venv_py' -m pip install -e .[test] --upgrade" Write-Host "Installing pipenv development packages" - Invoke-Expression "$venv_py -m pipenv install --dev" + Invoke-Expression "& '$venv_py' -m pipenv install --dev" Write-Host "Installing local package in pipenv environment" - Invoke-Expression "$venv_py -m pipenv run pip install -e .[test]" + Invoke-Expression "& '$venv_py' -m pipenv run pip install -e .[test]" Write-Host "Printing metadata" - Write-Host $(Invoke-Expression "$venv_py -m pipenv --venv") - Write-Host $(Invoke-Expression "$venv_py -m pipenv --py") - Write-Host $(Invoke-Expression "$venv_py -m pipenv run python --version") + Write-Host $(Invoke-Expression "& '$venv_py' -m pipenv --venv") + Write-Host $(Invoke-Expression "& '$venv_py' -m pipenv --py") + Write-Host $(Invoke-Expression "& '$venv_py' -m pipenv run python --version") displayName: Make Virtualenv env: PIPENV_DEFAULT_PYTHON_VERSION: $(PIPENV_DEFAULT_PYTHON_VERSION) diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index f33523f1..c3c7c1c6 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,8 +1,8 @@ steps: - powershell: | # Fix Git SSL errors - pip install certifi - python -m certifi > cacert.txt + Invoke-Expression "& '$VIRTUAL_ENV_PY' -m pip install certifi" + Invoke-Expression "& '$VIRTUAL_ENV_PY' -m certifi > cacert.txt" Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" $env:GIT_SSL_CAINFO="$(Get-Content cacert.txt)" # Shorten paths to get under MAX_PATH or else integration tests will fail @@ -14,8 +14,11 @@ steps: $env:TMP='T:\' git submodule sync git submodule update --init --recursive - D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests + Invoke-Expression "& '$VIRTUAL_ENV_PY' -m pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests" displayName: Run integration tests + env: + VIRTUAL_ENV: $(VIRTUAL_ENV) + VIRTUAL_ENV_PY: $(VIRTUAL_ENV_PY) - task: PublishTestResults@2 displayName: Publish Test Results diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 4f91350b..126dc809 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -185,10 +185,17 @@ def main(): sys.stderr = get_wrapped_stream(stderr) sys.stdout = get_wrapped_stream(stdout) from pipenv.vendor import colorama - colorama.init() - if os.name == "nt": - sys.stderr = colorama.AnsiToWin32(sys.stderr) - sys.stdout = colorama.AnsiToWin32(sys.stdout) + if os.name == "nt" and ( + all(getattr(stream, method, None) for stream in [sys.stdout, sys.stderr] for method in ["write", "isatty"]) and + all(stream.isatty() for stream in [sys.stdout, sys.stderr]) + ): + stderr_wrapper = colorama.AnsiToWin32(sys.stderr, autoreset=False, convert=None, strip=None) + stdout_wrapper = colorama.AnsiToWin32(sys.stdout, autoreset=False, convert=None, strip=None) + sys.stderr = stderr_wrapper.stream + sys.stdout = stdout_wrapper.stream + colorama.init(wrap=False) + elif os.name != "nt": + colorama.init() os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") os.environ["PYTHONIOENCODING"] = str("utf-8") parsed = handle_parsed_args(parsed) From a63fd73de839cbfcfa61210904422acfdefc9e3d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 20 Mar 2019 22:31:35 -0400 Subject: [PATCH 09/22] Fix azure pipelines invocation Signed-off-by: Dan Ryan --- .azure-pipelines/steps/run-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index c3c7c1c6..9c38d5e8 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,8 +1,8 @@ steps: - powershell: | # Fix Git SSL errors - Invoke-Expression "& '$VIRTUAL_ENV_PY' -m pip install certifi" - Invoke-Expression "& '$VIRTUAL_ENV_PY' -m certifi > cacert.txt" + Invoke-Expression "& '$env:VIRTUAL_ENV_PY' -m pip install certifi" + Invoke-Expression "& '$env:VIRTUAL_ENV_PY' -m certifi > cacert.txt" Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" $env:GIT_SSL_CAINFO="$(Get-Content cacert.txt)" # Shorten paths to get under MAX_PATH or else integration tests will fail @@ -14,7 +14,7 @@ steps: $env:TMP='T:\' git submodule sync git submodule update --init --recursive - Invoke-Expression "& '$VIRTUAL_ENV_PY' -m pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests" + Invoke-Expression "& '$env:VIRTUAL_ENV_PY' -m pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests" displayName: Run integration tests env: VIRTUAL_ENV: $(VIRTUAL_ENV) From c5729523ee2db66d76b81d53a97df3bf08cfb778 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 21 Mar 2019 01:16:42 -0400 Subject: [PATCH 10/22] Update vistir to rely on new rmtree logic Signed-off-by: Dan Ryan --- pipenv/patched/notpip/_internal/utils/temp_dir.py | 1 + pipenv/vendor/vistir/compat.py | 4 +--- tasks/vendoring/patches/patched/pip19.patch | 10 ++++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pipenv/patched/notpip/_internal/utils/temp_dir.py b/pipenv/patched/notpip/_internal/utils/temp_dir.py index d8911bc9..d8121a59 100644 --- a/pipenv/patched/notpip/_internal/utils/temp_dir.py +++ b/pipenv/patched/notpip/_internal/utils/temp_dir.py @@ -184,4 +184,5 @@ class AdjacentTempDirectory(TempDirectory): self.path = os.path.realpath( tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) ) + self._register_finalizer() logger.debug("Created temporary directory: {}".format(self.path)) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index e8688d89..d3f42e46 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -186,13 +186,11 @@ class TemporaryDirectory(object): os.unlink(path) # PermissionError is raised on FreeBSD for directories except (IsADirectoryError, PermissionError, OSError): - cls._rmtree(path) + rmtree(path) except FileNotFoundError: pass elif issubclass(exc_info[0], FileNotFoundError): pass - else: - raise rmtree(name, onerror=onerror) diff --git a/tasks/vendoring/patches/patched/pip19.patch b/tasks/vendoring/patches/patched/pip19.patch index 1c10d5c5..7dd912dc 100644 --- a/tasks/vendoring/patches/patched/pip19.patch +++ b/tasks/vendoring/patches/patched/pip19.patch @@ -289,7 +289,7 @@ index d1410e93..69a53bf2 100644 def cleanup_files(self): # type: () -> None diff --git a/pipenv/patched/notpip/_internal/resolve.py b/pipenv/patched/notpip/_internal/resolve.py -index 33f572f1..aa827193 100644 +index 33f572f1..dfe149ad 100644 --- a/pipenv/patched/notpip/_internal/resolve.py +++ b/pipenv/patched/notpip/_internal/resolve.py @@ -19,6 +19,7 @@ from pip._internal.exceptions import ( @@ -416,7 +416,7 @@ index 7aaf7b5e..d56f0512 100644 if not check_requires_python(requires_python): raise exceptions.UnsupportedPythonVersion( diff --git a/pipenv/patched/notpip/_internal/utils/temp_dir.py b/pipenv/patched/notpip/_internal/utils/temp_dir.py -index 2c81ad55..fbf0292a 100644 +index 2c81ad55..ff2ccc5a 100644 --- a/pipenv/patched/notpip/_internal/utils/temp_dir.py +++ b/pipenv/patched/notpip/_internal/utils/temp_dir.py @@ -5,8 +5,10 @@ import itertools @@ -485,6 +485,12 @@ index 2c81ad55..fbf0292a 100644 class AdjacentTempDirectory(TempDirectory): +@@ -152,4 +184,5 @@ class AdjacentTempDirectory(TempDirectory): + self.path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) + ) ++ self._register_finalizer() + logger.debug("Created temporary directory: {}".format(self.path)) diff --git a/pipenv/patched/notpip/_internal/wheel.py b/pipenv/patched/notpip/_internal/wheel.py index 67bcc7f7..968cdff9 100644 --- a/pipenv/patched/notpip/_internal/wheel.py From b77f6cfa95538142e60c913c855fc39609b60ac2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 21 Mar 2019 01:17:09 -0400 Subject: [PATCH 11/22] Point at new pip-tools version in patched file Signed-off-by: Dan Ryan --- pipenv/patched/patched.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index 41cb516e..e34df9fd 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -1,5 +1,5 @@ safety crayons==0.1.2 pipfile==0.0.2 -pip-tools==3.1.0 +pip-tools==3.5.0 pip==19.0.3 From 7d8f5172293c20717695a5e938e88f439ac675d1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 21 Mar 2019 22:05:32 -0400 Subject: [PATCH 12/22] Try ditching `pip_src_dir` in tests Signed-off-by: Dan Ryan --- tests/integration/test_install_uri.py | 28 +++++++++++++-------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index b0c1e51c..8e79670c 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -8,10 +8,10 @@ import delegator from pipenv._compat import Path +@flaky @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -@flaky def test_basic_vcs_install(PipenvInstance, pip_src_dir, pypi): with PipenvInstance(pypi=pypi, chdir=True) as p: c = p.pipenv("install git+https://github.com/benjaminp/six.git@1.11.0#egg=six") @@ -29,10 +29,10 @@ def test_basic_vcs_install(PipenvInstance, pip_src_dir, pypi): assert "gitdb2" in p.lockfile["default"] +@flaky @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -@flaky def test_git_vcs_install(PipenvInstance, pip_src_dir, pypi): with PipenvInstance(pypi=pypi, chdir=True) as p: c = p.pipenv("install git+git://github.com/benjaminp/six.git@1.11.0#egg=six") @@ -46,11 +46,11 @@ def test_git_vcs_install(PipenvInstance, pip_src_dir, pypi): } +@flaky @pytest.mark.vcs @pytest.mark.install -@pytest.mark.needs_github_ssh @pytest.mark.needs_internet -@flaky +@pytest.mark.needs_github_ssh def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi): with PipenvInstance(pypi=pypi, chdir=True) as p: c = p.pipenv("install git+ssh://git@github.com/benjaminp/six.git@1.11.0#egg=six") @@ -64,10 +64,10 @@ def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi): } -@pytest.mark.files -@pytest.mark.urls -@pytest.mark.needs_internet @flaky +@pytest.mark.urls +@pytest.mark.files +@pytest.mark.needs_internet def test_urls_work(PipenvInstance, pypi): with PipenvInstance(pypi=pypi, chdir=True) as p: # the library this installs is "django-cms" @@ -85,8 +85,8 @@ def test_urls_work(PipenvInstance, pypi): assert "file" in dep, p.lockfile -@pytest.mark.files @pytest.mark.urls +@pytest.mark.files def test_file_urls_work(PipenvInstance, pip_src_dir): with PipenvInstance(chdir=True) as p: whl = Path(__file__).parent.parent.joinpath( @@ -103,8 +103,8 @@ def test_file_urls_work(PipenvInstance, pip_src_dir): assert "file" in p.pipfile["packages"]["six"] -@pytest.mark.files @pytest.mark.urls +@pytest.mark.files @pytest.mark.needs_internet def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir): six_dir = tmpdir.join("six") @@ -124,7 +124,6 @@ def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir): @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -@flaky def test_editable_vcs_install(PipenvInstance, pip_src_dir, pypi): with PipenvInstance(pypi=pypi) as p: c = p.pipenv( @@ -141,12 +140,11 @@ def test_editable_vcs_install(PipenvInstance, pip_src_dir, pypi): assert "certifi" in p.lockfile["default"] -@pytest.mark.install @pytest.mark.vcs @pytest.mark.tablib +@pytest.mark.install @pytest.mark.needs_internet -@flaky -def test_install_editable_git_tag(PipenvInstance, pip_src_dir, pypi): +def test_install_editable_git_tag(PipenvInstance, pypi): # This uses the real PyPI since we need Internet to access the Git # dependency anyway. with PipenvInstance(pypi=pypi) as p: @@ -164,8 +162,8 @@ def test_install_editable_git_tag(PipenvInstance, pip_src_dir, pypi): assert "ref" in p.lockfile["default"]["six"] -@pytest.mark.install @pytest.mark.index +@pytest.mark.install @pytest.mark.needs_internet def test_install_named_index_alias(PipenvInstance): with PipenvInstance() as p: @@ -210,7 +208,7 @@ def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir): @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -def test_get_vcs_refs(PipenvInstance, pip_src_dir): +def test_get_vcs_refs(PipenvInstance): with PipenvInstance(chdir=True) as p: c = p.pipenv( "install -e git+https://github.com/benjaminp/six.git@1.9.0#egg=six" From 4f34b7c0d22108d46b63f61d712513745c2c0679 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 22 Mar 2019 20:27:52 -0400 Subject: [PATCH 13/22] Update vistir to fix permissionerrors Signed-off-by: Dan Ryan --- pipenv/vendor/vistir/compat.py | 25 +------------------------ pipenv/vendor/vistir/path.py | 12 +++++++----- 2 files changed, 8 insertions(+), 29 deletions(-) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index d3f42e46..ecc6bc26 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -169,30 +169,7 @@ class TemporaryDirectory(object): def _rmtree(cls, name): from .path import rmtree - def onerror(func, path, exc_info): - if issubclass(exc_info[0], (PermissionError, OSError)): - try: - try: - if path != name: - os.chflags(os.path.dirname(path), 0) - os.chflags(path, 0) - except AttributeError: - pass - if path != name: - os.chmod(os.path.dirname(path), 0o70) - os.chmod(path, 0o700) - - try: - os.unlink(path) - # PermissionError is raised on FreeBSD for directories - except (IsADirectoryError, PermissionError, OSError): - rmtree(path) - except FileNotFoundError: - pass - elif issubclass(exc_info[0], FileNotFoundError): - pass - - rmtree(name, onerror=onerror) + rmtree(name) @classmethod def _cleanup(cls, name, warn_message): diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 9eb11379..429c826e 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -20,6 +20,7 @@ from .compat import ( ResourceWarning, TemporaryDirectory, FileNotFoundError, + PermissionError, _fs_encoding, _NamedTemporaryFile, finalize, @@ -307,10 +308,11 @@ def set_write_bit(fn): file_stat = os.stat(fn).st_mode os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) if not os.path.isdir(fn): - try: - os.chflags(fn, 0) - except AttributeError: - pass + for path in [fn, os.path.dirname(fn)]: + try: + os.chflags(path, 0) + except AttributeError: + pass for root, dirs, files in os.walk(fn, topdown=False): for dir_ in [os.path.join(root, d) for d in dirs]: set_write_bit(dir_) @@ -338,7 +340,7 @@ def rmtree(directory, ignore_errors=False, onerror=None): onerror = handle_remove_readonly try: shutil.rmtree(directory, ignore_errors=ignore_errors, onerror=onerror) - except (IOError, OSError, FileNotFoundError) as exc: + except (IOError, OSError, FileNotFoundError, PermissionError) as exc: # Ignore removal failures where the file doesn't exist if exc.errno != errno.ENOENT: raise From 4bd02aa6cc11aea9c314eb15abd0cdb9cd963547 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 23 Mar 2019 17:38:20 -0400 Subject: [PATCH 14/22] Fix temporary directory cleanup Signed-off-by: Dan Ryan --- pipenv/patched/notpip/_internal/utils/misc.py | 4 +- pipenv/vendor/vistir/compat.py | 17 ++++- pipenv/vendor/vistir/path.py | 65 +++++++++++++++++-- 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/pipenv/patched/notpip/_internal/utils/misc.py b/pipenv/patched/notpip/_internal/utils/misc.py index a80153df..0a3237a2 100644 --- a/pipenv/patched/notpip/_internal/utils/misc.py +++ b/pipenv/patched/notpip/_internal/utils/misc.py @@ -117,8 +117,8 @@ def get_prog(): @retry(stop_max_delay=3000, wait_fixed=500) def rmtree(dir, ignore_errors=False): # type: (str, bool) -> None - shutil.rmtree(dir, ignore_errors=ignore_errors, - onerror=rmtree_errorhandler) + from pipenv.vendor.vistir.path import rmtree as vistir_rmtree, handle_remove_readonly + vistir_rmtree(dir, onerror=handle_remove_readonly, ignore_errors=ignore_errors) def rmtree_errorhandler(func, path, exc_info): diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index ecc6bc26..f0266e30 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -21,6 +21,8 @@ __all__ = [ "FileNotFoundError", "ResourceWarning", "PermissionError", + "is_type_checking", + "IS_TYPE_CHECKING", "IsADirectoryError", "fs_str", "lru_cache", @@ -132,8 +134,21 @@ if not sys.warnoptions: warnings.simplefilter("default", ResourceWarning) +def is_type_checking(): + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +IS_TYPE_CHECKING = is_type_checking() + + class TemporaryDirectory(object): - """Create and return a temporary directory. This has the same + + """ + Create and return a temporary directory. This has the same behavior as mkdtemp but can be used as a context manager. For example: diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 429c826e..e9370c8d 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -8,6 +8,7 @@ import os import posixpath import shutil import stat +import time import warnings import six @@ -26,8 +27,13 @@ from .compat import ( finalize, fs_decode, fs_encode, + IS_TYPE_CHECKING, ) +if IS_TYPE_CHECKING: + from typing import Optional, Callable, Text, ByteString, AnyStr + + __all__ = [ "check_for_unc_path", "get_converted_relative_path", @@ -89,6 +95,7 @@ else: def normalize_path(path): + # type: (AnyStr) -> AnyStr """ Return a case-normalized absolute variable-expanded path. @@ -105,6 +112,7 @@ def normalize_path(path): def is_in_path(path, parent): + # type: (AnyStr, AnyStr) -> bool """ Determine if the provided full path is in the given parent root. @@ -118,6 +126,7 @@ def is_in_path(path, parent): def normalize_drive(path): + # type: (str) -> Text """Normalize drive in path so they stay consistent. This currently only affects local drives on Windows, which can be @@ -138,6 +147,7 @@ def normalize_drive(path): def path_to_url(path): + # type: (str) -> Text """Convert the supplied local path to a file uri. :param str path: A string pointing to or representing a local path @@ -157,7 +167,9 @@ def path_to_url(path): def url_to_path(url): - """Convert a valid file url to a local filesystem path + # type: (str) -> ByteString + """ + Convert a valid file url to a local filesystem path Follows logic taken from pip's equivalent function """ @@ -296,10 +308,13 @@ def create_tracked_tempfile(*args, **kwargs): def set_write_bit(fn): - """Set read-write permissions for the current user on the target path. Fail silently + # type: (str) -> None + """ + Set read-write permissions for the current user on the target path. Fail silently if the path doesn't exist. :param str fn: The target filename or path + :return: None """ fn = fs_encode(fn) @@ -313,6 +328,7 @@ def set_write_bit(fn): os.chflags(path, 0) except AttributeError: pass + return None for root, dirs, files in os.walk(fn, topdown=False): for dir_ in [os.path.join(root, d) for d in dirs]: set_write_bit(dir_) @@ -321,7 +337,9 @@ def set_write_bit(fn): def rmtree(directory, ignore_errors=False, onerror=None): - """Stand-in for :func:`~shutil.rmtree` with additional error-handling. + # type: (str, bool, Optional[Callable]) -> None + """ + Stand-in for :func:`~shutil.rmtree` with additional error-handling. This version of `rmtree` handles read-only paths, especially in the case of index files written by certain source control systems. @@ -346,6 +364,39 @@ def rmtree(directory, ignore_errors=False, onerror=None): raise +def _wait_for_files(path): + """ + Retry with backoff up to 1 second to delete files from a directory. + + :param str path: The path to crawl to delete files from + :return: A list of remaining paths or None + :rtype: Optional[List[str]] + """ + timeout = 0.001 + remaining = [] + while timeout < 1.0: + remaining = [] + if os.path.isdir(path): + L = os.listdir(path) + for target in L: + _remaining = _wait_for_files(target) + if _remaining: + remaining.extend(_remaining) + continue + try: + os.unlink(path) + except FileNotFoundError as e: + if e.errno == errno.ENOENT: + return + except (OSError, IOError, PermissionError): + time.sleep(timeout) + timeout *= 2 + remaining.append(path) + else: + return + return remaining + + def handle_remove_readonly(func, path, exc): """Error handler for shutil.rmtree. @@ -375,11 +426,17 @@ def handle_remove_readonly(func, path, exc): if e.errno == errno.ENOENT: return elif e.errno in PERM_ERRORS: - warnings.warn(default_warning_message.format(path), ResourceWarning) + remaining = None + if os.path.isdir(path): + remaining =_wait_for_files(path) + if remaining: + warnings.warn(default_warning_message.format(path), ResourceWarning) return + raise if exc_exception.errno in PERM_ERRORS: set_write_bit(path) + remaining = _wait_for_files(path) try: func(path) except (OSError, IOError, FileNotFoundError, PermissionError) as e: From 4a0b2b759a7fac67b09090aae80fdad58c89e886 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 23 Mar 2019 21:59:44 -0400 Subject: [PATCH 15/22] monkeypatch shutil.rmtree for tests Signed-off-by: Dan Ryan --- tests/integration/conftest.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 446f5b4d..78fbd166 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,14 +1,19 @@ # -*- coding=utf-8 -*- +import errno import json import os +import shutil import sys import warnings +from functools import partial +from shutil import rmtree as _rmtree + import pytest -from vistir.compat import ResourceWarning, fs_str +from vistir.compat import ResourceWarning, fs_str, fs_encode, FileNotFoundError, PermissionError from vistir.contextmanagers import temp_environ -from vistir.path import mkdir_p, create_tracked_tempdir +from vistir.path import mkdir_p, create_tracked_tempdir, rmtree, handle_remove_readonly from pipenv._compat import Path, TemporaryDirectory from pipenv.exceptions import VirtualenvActivationException @@ -348,9 +353,24 @@ class _PipenvInstance(object): return os.sep.join([self.path, 'Pipfile.lock']) +def _rmtree_func(path, ignore_errors=False, onerror=None): + directory = fs_encode(path) + global _rmtree + shutil_rmtree = _rmtree + if onerror is None: + onerror = handle_remove_readonly + try: + shutil_rmtree(directory, ignore_errors=ignore_errors, onerror=onerror) + except (IOError, OSError, FileNotFoundError, PermissionError) as exc: + # Ignore removal failures where the file doesn't exist + if exc.errno != errno.ENOENT: + raise + + @pytest.fixture() -def PipenvInstance(): - with temp_environ(): +def PipenvInstance(monkeypatch): + with temp_environ(), monkeypatch.context() as m: + m.setattr(shutil, "rmtree", partial(_rmtree_func, ignore_errors=True)) original_umask = os.umask(0o007) os.environ["PIPENV_NOSPIN"] = fs_str("1") os.environ["CI"] = fs_str("1") From a9609901033f2e10f78e18db590cfe39b4b337dd Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 23 Mar 2019 22:36:03 -0400 Subject: [PATCH 16/22] fix double declaration of kwarg Signed-off-by: Dan Ryan --- tests/integration/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 78fbd166..b6fba7cf 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -353,7 +353,7 @@ class _PipenvInstance(object): return os.sep.join([self.path, 'Pipfile.lock']) -def _rmtree_func(path, ignore_errors=False, onerror=None): +def _rmtree_func(path, ignore_errors=True, onerror=None): directory = fs_encode(path) global _rmtree shutil_rmtree = _rmtree @@ -370,7 +370,7 @@ def _rmtree_func(path, ignore_errors=False, onerror=None): @pytest.fixture() def PipenvInstance(monkeypatch): with temp_environ(), monkeypatch.context() as m: - m.setattr(shutil, "rmtree", partial(_rmtree_func, ignore_errors=True)) + m.setattr(shutil, "rmtree", _rmtree_func) original_umask = os.umask(0o007) os.environ["PIPENV_NOSPIN"] = fs_str("1") os.environ["CI"] = fs_str("1") From 6971468f02a2d01c663c6e0b038905130b8f473a Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 24 Mar 2019 13:10:17 -0400 Subject: [PATCH 17/22] Write resolution results to temp json file instead of stdout Signed-off-by: Dan Ryan --- pipenv/core.py | 18 ++++++++++++++++++ pipenv/resolver.py | 26 ++++++++++++++++++-------- pipenv/utils.py | 25 ++++++++++++++++++++----- tests/integration/conftest.py | 16 ++++++++-------- 4 files changed, 64 insertions(+), 21 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 8e2059a6..e3c61618 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1314,6 +1314,11 @@ def pip_install( delete=False ) line = requirement.as_line(include_hashes=not ignore_hashes) + if environments.is_verbose(): + click.echo( + "Writing requirement line to temporary file: {0!r}".format(line), + err=True + ) f.write(vistir.misc.to_bytes(line)) r = f.name f.close() @@ -1380,6 +1385,8 @@ def pip_install( no_deps = False if src_dir is not None: + if environments.is_verbose(): + click.echo("Using source directory: {0!r}".format(src_dir)) repo = requirement.req.get_vcs_repo(src_dir=src_dir) else: repo = requirement.req.get_vcs_repo() @@ -1403,6 +1410,10 @@ def pip_install( line = "{0}&subdirectory={1}".format(line, repo.subdirectory) else: line = requirement.as_line(**line_kwargs) + click.echo( + "Writing requirement line to temporary file: {0!r}".format(line), + err=True + ) f.write(vistir.misc.to_bytes(line)) r = f.name f.close() @@ -1419,6 +1430,10 @@ def pip_install( ignore_hashes = True if not requirement.hashes else ignore_hashes line = requirement.as_line(include_hashes=not ignore_hashes) line = "{0} {1}".format(line, " ".join(src)) + click.echo( + "Writing requirement line to temporary file: {0!r}".format(line), + err=True + ) f.write(vistir.misc.to_bytes(line)) r = f.name f.close() @@ -2005,6 +2020,7 @@ def do_install( pypi_mirror=pypi_mirror, skip_lock=skip_lock, ) + pip_shims_module = os.environ.pop("PIP_SHIMS_BASE_MODULE", None) for pkg_line in pkg_list: click.echo( crayons.normal( @@ -2113,6 +2129,8 @@ def do_install( # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) + if pip_shims_module: + os.environ["PIP_SHIMS_BASE_MODULE"] = pip_shims_module do_init( dev=dev, system=system, diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 126dc809..c7314b22 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -65,6 +65,8 @@ def get_parser(): default=os.environ.get("PIPENV_SITE_DIR")) parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store", default=os.environ.get("PIPENV_REQ_DIR")) + parser.add_argument("--write", metavar="write", action="store", + default=os.environ.get("PIPENV_RESOLVER_FILE")) parser.add_argument("packages", nargs="*") return parser @@ -117,7 +119,7 @@ def parse_packages(packages, pre, clear, system, requirements_dir=None): print(json.dumps([])) -def resolve_packages(pre, clear, verbose, system, requirements_dir, packages): +def resolve_packages(pre, clear, verbose, system, write, requirements_dir, packages): from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources pypi_mirror_source = ( create_mirror_source(os.environ["PIPENV_PYPI_MIRROR"]) @@ -145,14 +147,21 @@ def resolve_packages(pre, clear, verbose, system, requirements_dir, packages): ) results = resolve(packages, pre=pre, project=project, sources=sources, clear=clear, system=system, requirements_dir=requirements_dir) - print("RESULTS:") - if results: - print(json.dumps(results)) + if write: + with open(write, "w") as fh: + if not results: + json.dump([], fh) + else: + json.dump(results, fh) else: - print(json.dumps([])) + print("RESULTS:") + if results: + print(json.dumps(results)) + else: + print(json.dumps([])) -def _main(pre, clear, verbose, system, requirements_dir, packages, parse_only=False): +def _main(pre, clear, verbose, system, write, requirements_dir, packages, parse_only=False): os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) os.environ["PIP_PYTHON_PATH"] = str(sys.executable) if parse_only: @@ -164,7 +173,7 @@ def _main(pre, clear, verbose, system, requirements_dir, packages, parse_only=Fa requirements_dir=requirements_dir, ) else: - resolve_packages(pre, clear, verbose, system, requirements_dir, packages) + resolve_packages(pre, clear, verbose, system, write, requirements_dir, packages) def main(): @@ -198,8 +207,9 @@ def main(): colorama.init() os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") os.environ["PYTHONIOENCODING"] = str("utf-8") + os.environ["PYTHONUNBUFFERED"] = str("1") parsed = handle_parsed_args(parsed) - _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, + _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.write, parsed.requirements_dir, parsed.packages, parse_only=parsed.parse_only) diff --git a/pipenv/utils.py b/pipenv/utils.py index e615faca..524374e0 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -951,7 +951,7 @@ def venv_resolve_deps( """ from .vendor.vistir.misc import fs_str - from .vendor.vistir.compat import Path, JSONDecodeError + from .vendor.vistir.compat import Path, JSONDecodeError, NamedTemporaryFile from .vendor.vistir.path import create_tracked_tempdir from . import resolver from ._compat import decode_for_output @@ -984,6 +984,9 @@ def venv_resolve_deps( cmd.append("--system") if dev: cmd.append("--dev") + target_file = NamedTemporaryFile(prefix="resolver", suffix=".json", delete=False) + target_file.close() + cmd.extend(["--write", make_posix(target_file.name)]) with temp_environ(): os.environ.update({fs_str(k): fs_str(val) for k, val in os.environ.items()}) if pypi_mirror: @@ -1007,15 +1010,27 @@ def venv_resolve_deps( c = resolve(cmd, sp) results = c.out.strip() sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) - if environments.is_verbose(): - click_echo(results.split("RESULTS:")[1], err=True) try: - results = json.loads(results.split("RESULTS:")[1].strip()) - + with open(target_file.name, "r") as fh: + results = json.load(fh) except (IndexError, JSONDecodeError): click_echo(c.out.strip(), err=True) click_echo(c.err.strip(), err=True) + if os.path.exists(target_file.name): + target_file.delete() raise RuntimeError("There was a problem with locking.") + if os.path.exists(target_file.name): + target_file.delete() + if environments.is_verbose(): + #click_echo(results.split("RESULTS:")[1], err=True) + click_echo(results, err=True) + # try: + # results = json.loads(results.split("RESULTS:")[1].strip()) + # except (IndexError, JSONDecodeError): + # click_echo(c.out.strip(), err=True) + # click_echo(c.err.strip(), err=True) + # raise RuntimeError("There was a problem with locking.") + if lockfile_section not in lockfile: lockfile[lockfile_section] = {} prepare_lockfile(results, pipfile, lockfile[lockfile_section]) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b6fba7cf..cd2b08e5 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,16 +6,15 @@ import shutil import sys import warnings -from functools import partial from shutil import rmtree as _rmtree import pytest -from vistir.compat import ResourceWarning, fs_str, fs_encode, FileNotFoundError, PermissionError +from vistir.compat import ResourceWarning, fs_str, fs_encode, FileNotFoundError, PermissionError, TemporaryDirectory from vistir.contextmanagers import temp_environ -from vistir.path import mkdir_p, create_tracked_tempdir, rmtree, handle_remove_readonly +from vistir.path import mkdir_p, create_tracked_tempdir, handle_remove_readonly -from pipenv._compat import Path, TemporaryDirectory +from pipenv._compat import Path from pipenv.exceptions import VirtualenvActivationException from pipenv.vendor import delegator, requests, toml, tomlkit from pytest_pypi.app import prepare_fixtures @@ -244,6 +243,7 @@ class _PipenvInstance(object): venv_root=None, ignore_virtualenvs=True, venv_in_project=True, name=None ): self.pypi = pypi + os.environ["PIPENV_VERBOSE"] = fs_str("1") if ignore_virtualenvs: os.environ["PIPENV_IGNORE_VIRTUALENVS"] = fs_str("1") if venv_root: @@ -328,8 +328,8 @@ class _PipenvInstance(object): # Pretty output for failing tests. if block: print('$ pipenv {0}'.format(cmd)) - print(c.out) - print(c.err) + print('Output: {0}'.format(c.out)) + print('Error: {0}'.format(c.err)) if c.return_code != 0: print("Command failed...") @@ -384,9 +384,9 @@ def PipenvInstance(monkeypatch): @pytest.fixture(autouse=True) -def pip_src_dir(request, pathlib_tmpdir): +def pip_src_dir(request, vistir_tmpdir): old_src_dir = os.environ.get('PIP_SRC', '') - os.environ['PIP_SRC'] = pathlib_tmpdir.as_posix() + os.environ['PIP_SRC'] = vistir_tmpdir.as_posix() def finalize(): os.environ['PIP_SRC'] = fs_str(old_src_dir) From 7ccaa3c75f01615f5dc4235581574e460fe0b0b2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 24 Mar 2019 14:07:34 -0400 Subject: [PATCH 18/22] Unlink file instead of calling non-existent delete function Signed-off-by: Dan Ryan --- pipenv/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 524374e0..20093959 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1017,10 +1017,10 @@ def venv_resolve_deps( click_echo(c.out.strip(), err=True) click_echo(c.err.strip(), err=True) if os.path.exists(target_file.name): - target_file.delete() + os.unlink(target_file.name) raise RuntimeError("There was a problem with locking.") if os.path.exists(target_file.name): - target_file.delete() + os.unlink(target_file.name) if environments.is_verbose(): #click_echo(results.split("RESULTS:")[1], err=True) click_echo(results, err=True) From 32e918fd46dd8a26d33bb7d72a2b66064adec9d7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 24 Mar 2019 14:49:15 -0400 Subject: [PATCH 19/22] Don't print extra strings to stdout Signed-off-by: Dan Ryan --- tests/integration/conftest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index cd2b08e5..fc113426 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,4 +1,5 @@ # -*- coding=utf-8 -*- +from __future__ import print_function import errno import json import os @@ -328,8 +329,8 @@ class _PipenvInstance(object): # Pretty output for failing tests. if block: print('$ pipenv {0}'.format(cmd)) - print('Output: {0}'.format(c.out)) - print('Error: {0}'.format(c.err)) + print(c.out) + print(c.err, file=sys.stderr) if c.return_code != 0: print("Command failed...") From 92010826c8fade133235ec4df900abb11c775b87 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 24 Mar 2019 15:58:10 -0400 Subject: [PATCH 20/22] Erm, remove verbose mode from tests Signed-off-by: Dan Ryan --- tests/integration/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index fc113426..766291e8 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -244,7 +244,6 @@ class _PipenvInstance(object): venv_root=None, ignore_virtualenvs=True, venv_in_project=True, name=None ): self.pypi = pypi - os.environ["PIPENV_VERBOSE"] = fs_str("1") if ignore_virtualenvs: os.environ["PIPENV_IGNORE_VIRTUALENVS"] = fs_str("1") if venv_root: From d9370508843b7f812bed1113f7fb8b7e3b422956 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 24 Mar 2019 23:30:13 -0400 Subject: [PATCH 21/22] Update pip patch Signed-off-by: Dan Ryan --- tasks/vendoring/patches/patched/pip19.patch | 81 ++++++++++++--------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/tasks/vendoring/patches/patched/pip19.patch b/tasks/vendoring/patches/patched/pip19.patch index 7dd912dc..74b98f50 100644 --- a/tasks/vendoring/patches/patched/pip19.patch +++ b/tasks/vendoring/patches/patched/pip19.patch @@ -1,7 +1,7 @@ -diff --git a/pipenv/patched/notpip/_internal/download.py b/pipenv/patched/notpip/_internal/download.py +diff --git a/pipenv/patched/pip/_internal/download.py b/pipenv/patched/pip/_internal/download.py index 2bbe1762..872af328 100644 ---- a/pipenv/patched/notpip/_internal/download.py -+++ b/pipenv/patched/notpip/_internal/download.py +--- a/pipenv/patched/pip/_internal/download.py ++++ b/pipenv/patched/pip/_internal/download.py @@ -77,7 +77,7 @@ def user_agent(): Return a string representing the user agent. """ @@ -11,10 +11,10 @@ index 2bbe1762..872af328 100644 "python": platform.python_version(), "implementation": { "name": platform.python_implementation(), -diff --git a/pipenv/patched/notpip/_internal/index.py b/pipenv/patched/notpip/_internal/index.py +diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py index 9eda3a35..67dd952c 100644 ---- a/pipenv/patched/notpip/_internal/index.py -+++ b/pipenv/patched/notpip/_internal/index.py +--- a/pipenv/patched/pip/_internal/index.py ++++ b/pipenv/patched/pip/_internal/index.py @@ -331,6 +331,9 @@ class PackageFinder(object): # The Session we'll use to make requests self.session = session @@ -156,10 +156,10 @@ index 9eda3a35..67dd952c 100644 def _find_name_version_sep(egg_info, canonical_name): -diff --git a/pipenv/patched/notpip/_internal/models/candidate.py b/pipenv/patched/notpip/_internal/models/candidate.py +diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py index 4475458a..6748957d 100644 ---- a/pipenv/patched/notpip/_internal/models/candidate.py -+++ b/pipenv/patched/notpip/_internal/models/candidate.py +--- a/pipenv/patched/pip/_internal/models/candidate.py ++++ b/pipenv/patched/pip/_internal/models/candidate.py @@ -13,11 +13,12 @@ class InstallationCandidate(KeyBasedCompareMixin): """Represents a potential "candidate" for installation. """ @@ -175,10 +175,10 @@ index 4475458a..6748957d 100644 super(InstallationCandidate, self).__init__( key=(self.project, self.version, self.location), -diff --git a/pipenv/patched/notpip/_internal/operations/prepare.py b/pipenv/patched/notpip/_internal/operations/prepare.py +diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py index 4f31dd5a..ed0c86b2 100644 ---- a/pipenv/patched/notpip/_internal/operations/prepare.py -+++ b/pipenv/patched/notpip/_internal/operations/prepare.py +--- a/pipenv/patched/pip/_internal/operations/prepare.py ++++ b/pipenv/patched/pip/_internal/operations/prepare.py @@ -17,7 +17,7 @@ from pip._internal.exceptions import ( from pip._internal.utils.compat import expanduser from pip._internal.utils.hashes import MissingHashes @@ -204,10 +204,10 @@ index 4f31dd5a..ed0c86b2 100644 req.populate_link(finder, upgrade_allowed, require_hashes) # We can't hit this spot and have populate_link return None. -diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py +diff --git a/pipenv/patched/pip/_internal/pep425tags.py b/pipenv/patched/pip/_internal/pep425tags.py index 1e782d1a..3c760ca3 100644 ---- a/pipenv/patched/notpip/_internal/pep425tags.py -+++ b/pipenv/patched/notpip/_internal/pep425tags.py +--- a/pipenv/patched/pip/_internal/pep425tags.py ++++ b/pipenv/patched/pip/_internal/pep425tags.py @@ -10,7 +10,10 @@ import sysconfig import warnings from collections import OrderedDict @@ -220,10 +220,10 @@ index 1e782d1a..3c760ca3 100644 from pip._internal.utils.compat import get_extension_suffixes from pip._internal.utils.typing import MYPY_CHECK_RUNNING -diff --git a/pipenv/patched/notpip/_internal/req/req_install.py b/pipenv/patched/notpip/_internal/req/req_install.py +diff --git a/pipenv/patched/pip/_internal/req/req_install.py b/pipenv/patched/pip/_internal/req/req_install.py index a4834b00..2c22e141 100644 ---- a/pipenv/patched/notpip/_internal/req/req_install.py -+++ b/pipenv/patched/notpip/_internal/req/req_install.py +--- a/pipenv/patched/pip/_internal/req/req_install.py ++++ b/pipenv/patched/pip/_internal/req/req_install.py @@ -588,7 +588,8 @@ class InstallRequirement(object): self.setup_py, self.link, ) @@ -256,10 +256,10 @@ index a4834b00..2c22e141 100644 install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ -diff --git a/pipenv/patched/notpip/_internal/req/req_set.py b/pipenv/patched/notpip/_internal/req/req_set.py +diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py index d1410e93..69a53bf2 100644 ---- a/pipenv/patched/notpip/_internal/req/req_set.py -+++ b/pipenv/patched/notpip/_internal/req/req_set.py +--- a/pipenv/patched/pip/_internal/req/req_set.py ++++ b/pipenv/patched/pip/_internal/req/req_set.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) class RequirementSet(object): @@ -288,10 +288,10 @@ index d1410e93..69a53bf2 100644 def cleanup_files(self): # type: () -> None -diff --git a/pipenv/patched/notpip/_internal/resolve.py b/pipenv/patched/notpip/_internal/resolve.py +diff --git a/pipenv/patched/pip/_internal/resolve.py b/pipenv/patched/pip/_internal/resolve.py index 33f572f1..dfe149ad 100644 ---- a/pipenv/patched/notpip/_internal/resolve.py -+++ b/pipenv/patched/notpip/_internal/resolve.py +--- a/pipenv/patched/pip/_internal/resolve.py ++++ b/pipenv/patched/pip/_internal/resolve.py @@ -19,6 +19,7 @@ from pip._internal.exceptions import ( UnsupportedPythonVersion, ) @@ -389,10 +389,10 @@ index 33f572f1..dfe149ad 100644 if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took -diff --git a/pipenv/patched/notpip/_internal/utils/packaging.py b/pipenv/patched/notpip/_internal/utils/packaging.py +diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py index 7aaf7b5e..d56f0512 100644 ---- a/pipenv/patched/notpip/_internal/utils/packaging.py -+++ b/pipenv/patched/notpip/_internal/utils/packaging.py +--- a/pipenv/patched/pip/_internal/utils/packaging.py ++++ b/pipenv/patched/pip/_internal/utils/packaging.py @@ -37,7 +37,7 @@ def check_requires_python(requires_python): requires_python_specifier = specifiers.SpecifierSet(requires_python) @@ -415,10 +415,10 @@ index 7aaf7b5e..d56f0512 100644 try: if not check_requires_python(requires_python): raise exceptions.UnsupportedPythonVersion( -diff --git a/pipenv/patched/notpip/_internal/utils/temp_dir.py b/pipenv/patched/notpip/_internal/utils/temp_dir.py +diff --git a/pipenv/patched/pip/_internal/utils/temp_dir.py b/pipenv/patched/pip/_internal/utils/temp_dir.py index 2c81ad55..ff2ccc5a 100644 ---- a/pipenv/patched/notpip/_internal/utils/temp_dir.py -+++ b/pipenv/patched/notpip/_internal/utils/temp_dir.py +--- a/pipenv/patched/pip/_internal/utils/temp_dir.py ++++ b/pipenv/patched/pip/_internal/utils/temp_dir.py @@ -5,8 +5,10 @@ import itertools import logging import os.path @@ -491,10 +491,10 @@ index 2c81ad55..ff2ccc5a 100644 ) + self._register_finalizer() logger.debug("Created temporary directory: {}".format(self.path)) -diff --git a/pipenv/patched/notpip/_internal/wheel.py b/pipenv/patched/notpip/_internal/wheel.py +diff --git a/pipenv/patched/pip/_internal/wheel.py b/pipenv/patched/pip/_internal/wheel.py index 67bcc7f7..968cdff9 100644 ---- a/pipenv/patched/notpip/_internal/wheel.py -+++ b/pipenv/patched/notpip/_internal/wheel.py +--- a/pipenv/patched/pip/_internal/wheel.py ++++ b/pipenv/patched/pip/_internal/wheel.py @@ -114,7 +114,7 @@ def fix_script(path): firstline = script.readline() if not firstline.startswith(b'#!python'): @@ -525,3 +525,18 @@ index 67bcc7f7..968cdff9 100644 SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) +diff --git a/pipenv/patched/pip/_internal/utils/misc.py b/pipenv/patched/pip/_internal/utils/misc.py +index 84605ee3..649311c0 100644 +--- a/pipenv/patched/pip/_internal/utils/misc.py ++++ b/pipenv/patched/pip/_internal/utils/misc.py +@@ -117,8 +117,8 @@ def get_prog(): + @retry(stop_max_delay=3000, wait_fixed=500) + def rmtree(dir, ignore_errors=False): + # type: (str, bool) -> None +- shutil.rmtree(dir, ignore_errors=ignore_errors, +- onerror=rmtree_errorhandler) ++ from pipenv.vendor.vistir.path import rmtree as vistir_rmtree, handle_remove_readonly ++ vistir_rmtree(dir, onerror=handle_remove_readonly, ignore_errors=ignore_errors) + + + def rmtree_errorhandler(func, path, exc_info): From 4cc36083f14b51337cec65912b4b857343b631c2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 25 Mar 2019 00:13:02 -0400 Subject: [PATCH 22/22] Clean up old code Signed-off-by: Dan Ryan --- pipenv/utils.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 20093959..252ed30f 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1022,14 +1022,7 @@ def venv_resolve_deps( if os.path.exists(target_file.name): os.unlink(target_file.name) if environments.is_verbose(): - #click_echo(results.split("RESULTS:")[1], err=True) click_echo(results, err=True) - # try: - # results = json.loads(results.split("RESULTS:")[1].strip()) - # except (IndexError, JSONDecodeError): - # click_echo(c.out.strip(), err=True) - # click_echo(c.err.strip(), err=True) - # raise RuntimeError("There was a problem with locking.") if lockfile_section not in lockfile: lockfile[lockfile_section] = {}