From c42cf75f4a2ad5911bfecea0f0e8a74720c9fea7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 3 Nov 2018 00:36:02 -0400 Subject: [PATCH] Performance improvements Signed-off-by: Dan Ryan --- .../notpip/_internal/cli/autocompletion.py | 7 +- .../notpip/_internal/cli/base_command.py | 47 +++-- .../notpip/_internal/cli/main_parser.py | 21 +- .../notpip/_internal/commands/__init__.py | 196 ++++++++++++------ pipenv/vendor/requirementslib/models/cache.py | 4 +- .../requirementslib/models/requirements.py | 10 +- .../requirementslib/models/resolvers.py | 5 +- pipenv/vendor/requirementslib/models/utils.py | 10 +- pipenv/vendor/requirementslib/models/vcs.py | 8 +- pipenv/vendor/requirementslib/utils.py | 120 +++++++---- 10 files changed, 258 insertions(+), 170 deletions(-) diff --git a/pipenv/patched/notpip/_internal/cli/autocompletion.py b/pipenv/patched/notpip/_internal/cli/autocompletion.py index 15b560a1..14022cae 100644 --- a/pipenv/patched/notpip/_internal/cli/autocompletion.py +++ b/pipenv/patched/notpip/_internal/cli/autocompletion.py @@ -5,12 +5,11 @@ import optparse import os import sys -from pipenv.patched.notpip._internal.cli.main_parser import create_main_parser -from pipenv.patched.notpip._internal.commands import commands_dict, get_summaries -from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions - def autocomplete(): + from pipenv.patched.notpip._internal.cli.main_parser import create_main_parser + from pipenv.patched.notpip._internal.commands import commands_dict, get_summaries + from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions """Entry Point for completion of main and subcommand options. """ # Don't complete if user hasn't sourced bash_completion file. diff --git a/pipenv/patched/notpip/_internal/cli/base_command.py b/pipenv/patched/notpip/_internal/cli/base_command.py index 229831f2..8e8a84f4 100644 --- a/pipenv/patched/notpip/_internal/cli/base_command.py +++ b/pipenv/patched/notpip/_internal/cli/base_command.py @@ -7,28 +7,6 @@ import optparse import os import sys -from pipenv.patched.notpip._internal.cli import cmdoptions -from pipenv.patched.notpip._internal.cli.parser import ( - ConfigOptionParser, UpdatingDefaultsHelpFormatter, -) -from pipenv.patched.notpip._internal.cli.status_codes import ( - ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, - VIRTUALENV_NOT_FOUND, -) -from pipenv.patched.notpip._internal.download import PipSession -from pipenv.patched.notpip._internal.exceptions import ( - BadCommand, CommandError, InstallationError, PreviousBuildDirError, - UninstallationError, -) -from pipenv.patched.notpip._internal.index import PackageFinder -from pipenv.patched.notpip._internal.locations import running_under_virtualenv -from pipenv.patched.notpip._internal.req.constructors import ( - install_req_from_editable, install_req_from_line, -) -from pipenv.patched.notpip._internal.req.req_file import parse_requirements -from pipenv.patched.notpip._internal.utils.logging import setup_logging -from pipenv.patched.notpip._internal.utils.misc import get_prog, normalize_path -from pipenv.patched.notpip._internal.utils.outdated import pip_version_check from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: @@ -46,6 +24,11 @@ class Command(object): ignore_require_venv = False # type: bool def __init__(self, isolated=False): + from pipenv.patched.notpip._internal.utils.misc import get_prog + from pipenv.patched.notpip._internal.cli import cmdoptions + from pipenv.patched.notpip._internal.cli.parser import ( + ConfigOptionParser, UpdatingDefaultsHelpFormatter + ) parser_kw = { 'usage': self.usage, 'prog': '%s %s' % (get_prog(), self.name), @@ -70,6 +53,8 @@ class Command(object): self.parser.add_option_group(gen_opts) def _build_session(self, options, retries=None, timeout=None): + from pipenv.patched.notpip._internal.download import PipSession + from pipenv.patched.notpip._internal.utils.misc import normalize_path session = PipSession( cache=( normalize_path(os.path.join(options.cache_dir, "http")) @@ -110,6 +95,16 @@ class Command(object): return self.parser.parse_args(args) def main(self, args): + + from pipenv.patched.notpip._internal.cli.status_codes import ( + ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, + VIRTUALENV_NOT_FOUND, + ) + from pipenv.patched.notpip._internal.exceptions import ( + BadCommand, CommandError, InstallationError, PreviousBuildDirError, + UninstallationError, + ) + from pipenv.patched.notpip._internal.utils.logging import setup_logging options, args = self.parse_args(args) # Set verbosity so that it can be used elsewhere. @@ -132,6 +127,7 @@ class Command(object): os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action) if options.require_venv and not self.ignore_require_venv: + from pipenv.patched.notpip._internal.locations import running_under_virtualenv # If a venv is required check if it can really be found if not running_under_virtualenv(): logger.critical( @@ -184,6 +180,7 @@ class Command(object): timeout=min(5, options.timeout) ) with session: + from pipenv.patched.notpip._internal.utils.outdated import pip_version_check pip_version_check(session, options) # Shutdown the logging module @@ -200,6 +197,11 @@ class RequirementCommand(Command): """ Marshal cmd line args into a requirement set. """ + from pipenv.patched.notpip._internal.exceptions import CommandError + from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, + ) + from pipenv.patched.notpip._internal.req.req_file import parse_requirements # NOTE: As a side-effect, options.require_hashes and # requirement_set.require_hashes may be updated @@ -257,6 +259,7 @@ class RequirementCommand(Command): """ Create a package finder appropriate to this requirement command. """ + from pipenv.patched.notpip._internal.index import PackageFinder index_urls = [options.index_url] + options.extra_index_urls if options.no_index: logger.debug('Ignoring indexes: %s', ','.join(index_urls)) diff --git a/pipenv/patched/notpip/_internal/cli/main_parser.py b/pipenv/patched/notpip/_internal/cli/main_parser.py index abe2f69e..d87b733f 100644 --- a/pipenv/patched/notpip/_internal/cli/main_parser.py +++ b/pipenv/patched/notpip/_internal/cli/main_parser.py @@ -4,16 +4,6 @@ import os import sys -from pipenv.patched.notpip import __version__ -from pipenv.patched.notpip._internal.cli import cmdoptions -from pipenv.patched.notpip._internal.cli.parser import ( - ConfigOptionParser, UpdatingDefaultsHelpFormatter, -) -from pipenv.patched.notpip._internal.commands import ( - commands_dict, get_similar_commands, get_summaries, -) -from pipenv.patched.notpip._internal.exceptions import CommandError -from pipenv.patched.notpip._internal.utils.misc import get_prog __all__ = ["create_main_parser", "parse_command"] @@ -22,6 +12,13 @@ def create_main_parser(): """Creates and returns the main parser for pip's CLI """ + from pipenv.patched.notpip import __version__ + from pipenv.patched.notpip._internal.cli import cmdoptions + from pipenv.patched.notpip._internal.cli.parser import ( + ConfigOptionParser, UpdatingDefaultsHelpFormatter, + ) + from pipenv.patched.notpip._internal.commands import get_summaries + from pipenv.patched.notpip._internal.utils.misc import get_prog parser_kw = { 'usage': '\n%prog [options]', 'add_help_option': False, @@ -55,6 +52,10 @@ def create_main_parser(): def parse_command(args): + from pipenv.patched.notpip._internal.commands import ( + commands_dict, get_similar_commands + ) + from pipenv.patched.notpip._internal.exceptions import CommandError parser = create_main_parser() # Note: parser calls disable_interspersed_args(), so the result of this diff --git a/pipenv/patched/notpip/_internal/commands/__init__.py b/pipenv/patched/notpip/_internal/commands/__init__.py index a403c6f9..2e81aa2e 100644 --- a/pipenv/patched/notpip/_internal/commands/__init__.py +++ b/pipenv/patched/notpip/_internal/commands/__init__.py @@ -2,78 +2,136 @@ Package containing all pip commands """ from __future__ import absolute_import - -from pipenv.patched.notpip._internal.commands.completion import CompletionCommand -from pipenv.patched.notpip._internal.commands.configuration import ConfigurationCommand -from pipenv.patched.notpip._internal.commands.download import DownloadCommand -from pipenv.patched.notpip._internal.commands.freeze import FreezeCommand -from pipenv.patched.notpip._internal.commands.hash import HashCommand -from pipenv.patched.notpip._internal.commands.help import HelpCommand -from pipenv.patched.notpip._internal.commands.list import ListCommand -from pipenv.patched.notpip._internal.commands.check import CheckCommand -from pipenv.patched.notpip._internal.commands.search import SearchCommand -from pipenv.patched.notpip._internal.commands.show import ShowCommand -from pipenv.patched.notpip._internal.commands.install import InstallCommand -from pipenv.patched.notpip._internal.commands.uninstall import UninstallCommand -from pipenv.patched.notpip._internal.commands.wheel import WheelCommand - -from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING - -if MYPY_CHECK_RUNNING: - from typing import List, Type # noqa: F401 - from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 - -commands_order = [ - InstallCommand, - DownloadCommand, - UninstallCommand, - FreezeCommand, - ListCommand, - ShowCommand, - CheckCommand, - ConfigurationCommand, - SearchCommand, - WheelCommand, - HashCommand, - CompletionCommand, - HelpCommand, -] # type: List[Type[Command]] - -commands_dict = {c.name: c for c in commands_order} +import importlib +import sys -def get_summaries(ordered=True): - """Yields sorted (command name, command summary) tuples.""" +class _command(object): - if ordered: - cmditems = _sort_commands(commands_dict, commands_order) - else: - cmditems = commands_dict.items() + def __dir__(self): + result = list(self._locations.keys()) + list(self.__dict__.keys()) + result.extend(('__file__', '__doc__', '__all__', + '__docformat__', '__name__', '__path__', + '__package__', '__version__')) + return result - for name, command_class in cmditems: - yield (name, command_class.summary) + @property + def __all__(self): + return self._commands_order + ["get_summaries", "get_similar_commands"] + + @classmethod + def _new(cls): + return cls() + + def __init__(self): + self._modules = { + "sys": sys, + } + self._module_paths = {} + self._cached_commands_order = [] + self._commands_order = [ + "pipenv.patched.notpip._internal.commands.install.InstallCommand", + "pipenv.patched.notpip._internal.commands.download.DownloadCommand", + "pipenv.patched.notpip._internal.commands.uninstall.UninstallCommand", + "pipenv.patched.notpip._internal.commands.freeze.FreezeCommand", + "pipenv.patched.notpip._internal.commands.list.ListCommand", + "pipenv.patched.notpip._internal.commands.show.ShowCommand", + "pipenv.patched.notpip._internal.commands.check.CheckCommand", + "pipenv.patched.notpip._internal.commands.configuration.ConfigurationCommand", + "pipenv.patched.notpip._internal.commands.search.SearchCommand", + "pipenv.patched.notpip._internal.commands.wheel.WheelCommand", + "pipenv.patched.notpip._internal.commands.hash.HashCommand", + "pipenv.patched.notpip._internal.commands.completion.CompletionCommand", + "pipenv.patched.notpip._internal.commands.help.HelpCommand", + ] + for cmd in self._commands_order: + _, _, cmdname = cmd.rpartition(".") + self._module_paths[cmdname] = cmd + self._commands_dict = {} + + @property + def commands_order(self): + if not self._cached_commands_order: + commands = [self.get_package(cmd) for cmd in self._commands_order] + self._cached_commands_order = [getattr(self, cmd) for _, cmd in commands] + return self._cached_commands_order + + @property + def commands_dict(self): + if not self._commands_dict: + self._commands_dict = {c.name: c for c in self.commands_order} + return self._commands_dict + + def __getattr__(self, key): + modules = super(_command, self).__getattribute__("_modules") + module_paths = super(_command, self).__getattribute__("_module_paths") + if key in modules: + return modules[key] + elif key in self._module_paths: + module = self._import(module_paths[key]) + if module: + self._modules[key] = module + return module + return super(_command, self).__getattribute__(key) + + def get_package(self, module, subimport=None): + package = None + if subimport: + package = subimport + else: + module, _, package = module.rpartition(".") + return module, package + + def get_package_from_module(self, module): + module, package = self.get_package(module) + mod = importlib.import_module(module) + pkg = getattr(mod, package, None) + return pkg + + def _import(self, package): + return self.get_package_from_module(module) + + def get_summaries(self, ordered=True): + """Yields sorted (command name, command summary) tuples.""" + + if ordered: + cmditems = self._sort_commands(self.commands_dict, self.commands_order) + else: + cmditems = self.commands_dict.items() + + for name, command_class in cmditems: + yield (name, command_class.summary) + + def get_similar_commands(self, name): + """Command name auto-correct.""" + from difflib import get_close_matches + + name = name.lower() + + close_commands = get_close_matches(name, self.commands_dict.keys()) + + if close_commands: + return close_commands[0] + else: + return False + + def _sort_commands(self, cmddict, order): + def keyfn(key): + try: + return order.index(key[1]) + except ValueError: + # unordered items should come last + return 0xff + + return sorted(cmddict.items(), key=keyfn) -def get_similar_commands(name): - """Command name auto-correct.""" - from difflib import get_close_matches - - name = name.lower() - - close_commands = get_close_matches(name, commands_dict.keys()) - - if close_commands: - return close_commands[0] - else: - return False - - -def _sort_commands(cmddict, order): - def keyfn(key): - try: - return order.index(key[1]) - except ValueError: - # unordered items should come last - return 0xff - - return sorted(cmddict.items(), key=keyfn) +old_module = sys.modules[__name__] if __name__ in sys.modules else None +module = sys.modules[__name__] = _command() +module.__dict__.update({ + '__file__': __file__, + '__package__': __package__, + '__doc__': __doc__, + '__all__': module.__all__, + '__name__': __name__, +}) diff --git a/pipenv/vendor/requirementslib/models/cache.py b/pipenv/vendor/requirementslib/models/cache.py index 71701090..3e08c485 100644 --- a/pipenv/vendor/requirementslib/models/cache.py +++ b/pipenv/vendor/requirementslib/models/cache.py @@ -17,7 +17,6 @@ from packaging.requirements import Requirement from .utils import as_tuple, key_from_req, lookup_table, get_pinned_version from ..exceptions import FileExistsError -from ..utils import VCS_SUPPORT CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) @@ -202,9 +201,10 @@ class HashCache(SafeFileCache): super(HashCache, self).__init__(*args, **kwargs) def get_hash(self, location): + from pip_shims import VcsSupport # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs = VCS_SUPPORT + vcs = VcsSupport() orig_scheme = location.scheme new_location = copy.deepcopy(location) if orig_scheme in vcs.all_schemes: diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 112791ab..f59c7a4f 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -9,14 +9,13 @@ import os from contextlib import contextmanager import attr -import six from first import first from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement from packaging.specifiers import Specifier, SpecifierSet from packaging.utils import canonicalize_name -from pip_shims.shims import _strip_extras, parse_version, path_to_url, url_to_path, Link +from pip_shims.shims import _strip_extras, parse_version, path_to_url, url_to_path from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote from vistir.compat import FileNotFoundError, Path @@ -29,15 +28,13 @@ from vistir.path import ( from ..exceptions import RequirementError from ..utils import VCS_LIST, is_installable_file, is_vcs, ensure_setup_py from .baserequirement import BaseRequirement -from .markers import PipenvMarkers from .utils import ( HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string, filter_none, format_requirement, get_version, init_requirement, is_pinned_requirement, make_install_requirement, optional_instance_of, parse_extras, specs_to_string, split_markers_from_line, ireq_from_editable, ireq_from_line, split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs, normalize_name, create_link, - Requirement as PkgResourcesRequirement + validate_specifiers, validate_vcs, normalize_name, create_link ) @@ -45,7 +42,7 @@ from .utils import ( class NamedRequirement(BaseRequirement): name = attr.ib() version = attr.ib(validator=attr.validators.optional(validate_specifiers)) - req = attr.ib(type=PkgResourcesRequirement) + req = attr.ib() extras = attr.ib(default=attr.Factory(list)) editable = attr.ib(default=False) @@ -935,6 +932,7 @@ class Requirement(object): @classmethod def from_pipfile(cls, name, pipfile): + from .markers import PipenvMarkers _pipfile = {} if hasattr(pipfile, "keys"): _pipfile = dict(pipfile).copy() diff --git a/pipenv/vendor/requirementslib/models/resolvers.py b/pipenv/vendor/requirementslib/models/resolvers.py index 1a239390..bd773ba6 100644 --- a/pipenv/vendor/requirementslib/models/resolvers.py +++ b/pipenv/vendor/requirementslib/models/resolvers.py @@ -6,7 +6,6 @@ import six from pip_shims.shims import Wheel -from ..utils import log, VCS_SUPPORT from .cache import HashCache from .utils import format_requirement, is_pinned_requirement, version_from_ireq @@ -141,6 +140,7 @@ class DependencyResolver(object): # Coerce input into AbstractDependency instances. # We accept str, Requirement, and AbstractDependency as input. from .dependencies import AbstractDependency + from ..utils import log for dep in root_nodes: if isinstance(dep, six.string_types): dep = AbstractDependency.from_string(dep) @@ -193,7 +193,8 @@ class DependencyResolver(object): if ireq.editable: return set() - vcs = VCS_SUPPORT + from pip_shims import VcsSupport + vcs = VcsSupport() if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: return set() diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index d72542c4..84f9a92c 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -14,10 +14,6 @@ from attr import validators from first import first from packaging.markers import InvalidMarker, Marker, Op, Value, Variable from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet -from packaging.version import parse as parse_version -from packaging.requirements import Requirement as PackagingRequirement -from pkg_resources import Requirement - from vistir.misc import dedup @@ -53,6 +49,7 @@ def ireq_from_editable(ireq): def init_requirement(name): + from pkg_resources import Requirement req = Requirement.parse(name) req.vcs = None req.local_file = None @@ -74,6 +71,7 @@ def extras_to_string(extras): def parse_extras(extras_str): """Turn a string of extras into a parsed extras list""" + from pkg_resources import Requirement extras = Requirement.parse("fakepkg{0}".format(extras_to_string(extras_str))).extras return sorted(dedup([extra.lower() for extra in extras])) @@ -132,7 +130,7 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pipenv.patched.notpip format""" + """Cleans VCS uris from pip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: @@ -483,6 +481,7 @@ def clean_requires_python(candidates): """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" all_candidates = [] sys_version = '.'.join(map(str, sys.version_info[:3])) + from packaging.version import parse as parse_version py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', sys_version)) for c in candidates: from_location = attrgetter("location.requires_python") @@ -504,6 +503,7 @@ def clean_requires_python(candidates): def fix_requires_python_marker(requires_python): + from packaging.requirements import Requirement as PackagingRequirement marker_str = '' if any(requires_python.startswith(op) for op in Specifier._operators.keys()): spec_dict = defaultdict(set) diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index 4efb9bd3..dd8cc3a4 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -1,11 +1,9 @@ # -*- coding=utf-8 -*- import attr -from pip_shims import VcsSupport, parse_version, pip_version import os +import pip_shims -VCS_SUPPORT = VcsSupport() - @attr.s class VCSRepository(object): @@ -20,6 +18,8 @@ class VCSRepository(object): @repo_instance.default def get_repo_instance(self): + from pip_shims import VcsSupport + VCS_SUPPORT = VcsSupport() backend = VCS_SUPPORT._registry.get(self.vcs_type) return backend(url=self.url) @@ -51,7 +51,7 @@ class VCSRepository(object): def update(self, ref): target_ref = self.repo_instance.make_rev_options(ref) - if parse_version(pip_version) > parse_version("18.0"): + if pip_shims.parse_version(pip_shims.pip_version) > pip_shims.parse_version("18.0"): self.repo_instance.update(self.checkout_directory, self.url, target_ref) else: self.repo_instance.update(self.checkout_directory, target_ref) diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index 291c30ae..f1fc1337 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -16,22 +16,12 @@ six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) from six.moves import Mapping, Sequence, Set, ItemsView from six.moves.urllib.parse import urlparse, urlsplit -from pip_shims.shims import ( - Command, VcsSupport, cmdoptions, is_archive_file, - is_installable_dir as _is_installable_dir -) +import pip_shims from vistir.compat import Path from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir VCS_LIST = ("git", "svn", "hg", "bzr") -VCS_SCHEMES = [] -SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") - -VCS_SUPPORT = VcsSupport() - -if not VCS_SCHEMES: - VCS_SCHEMES = VCS_SUPPORT.all_schemes def setup_logger(): @@ -47,8 +37,38 @@ def setup_logger(): log = setup_logger() +SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") + + +VCS_SCHEMES = [ + "git", + "git+http", + "git+https", + "git+ssh", + "git+git", + "git+file", + "hg", + "hg+http", + "hg+https", + "hg+ssh", + "hg+static-http", + "svn", + "svn+ssh", + "svn+http", + "svn+https", + "svn+svn", + "bzr", + "bzr+http", + "bzr+https", + "bzr+ssh", + "bzr+sftp", + "bzr+ftp", + "bzr+lp", +] + + def is_installable_dir(path): - if _is_installable_dir(path): + if pip_shims.shims.is_installable_dir(path): return True path = Path(path) pyproject = path.joinpath("pyproject.toml") @@ -68,6 +88,7 @@ def is_vcs(pipfile_entry): elif isinstance(pipfile_entry, six.string_types): if not is_valid_url(pipfile_entry) and pipfile_entry.startswith("git+"): from .models.utils import add_ssh_scheme_to_git_uri + pipfile_entry = add_ssh_scheme_to_git_uri(pipfile_entry) parsed_entry = urlsplit(pipfile_entry) return parsed_entry.scheme in VCS_SCHEMES @@ -125,7 +146,7 @@ def is_installable_file(path): if lookup_path.is_dir() and is_installable_dir(absolute_path): return True - elif lookup_path.is_file() and is_archive_file(absolute_path): + elif lookup_path.is_file() and pip_shims.shims.is_archive_file(absolute_path): return True return False @@ -139,9 +160,7 @@ def prepare_pip_source_args(sources, pip_args=None): pip_args.extend(["-i", sources[0]["url"]]) # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): - pip_args.extend( - ["--trusted-host", urlparse(sources[0]["url"]).hostname] - ) + pip_args.extend(["--trusted-host", urlparse(sources[0]["url"]).hostname]) # Add additional sources as extra indexes. if len(sources) > 1: for source in sources[1:]: @@ -154,24 +173,25 @@ def prepare_pip_source_args(sources, pip_args=None): return pip_args -class PipCommand(Command): - name = 'PipCommand' - - def get_pip_command(): # Use pip's parser for pip.conf management and defaults. # General options (find_links, index_url, extra_index_url, trusted_host, # and pre) are defered to pip. import optparse + + class PipCommand(pip_shims.shims.Command): + name = "PipCommand" + pip_command = PipCommand() - pip_command.parser.add_option(cmdoptions.no_binary()) - pip_command.parser.add_option(cmdoptions.only_binary()) - index_opts = cmdoptions.make_option_group( - cmdoptions.index_group, - pip_command.parser, + pip_command.parser.add_option(pip_shims.shims.cmdoptions.no_binary()) + pip_command.parser.add_option(pip_shims.shims.cmdoptions.only_binary()) + index_opts = pip_shims.shims.cmdoptions.make_option_group( + pip_shims.shims.cmdoptions.index_group, pip_command.parser ) pip_command.parser.insert_option_group(0, index_opts) - pip_command.parser.add_option(optparse.Option('--pre', action='store_true', default=False)) + pip_command.parser.add_option( + optparse.Option("--pre", action="store_true", default=False) + ) return pip_command @@ -204,7 +224,6 @@ def ensure_setup_py(base_dir): setup_py.unlink() - _UNSET = object() _REMAP_EXIT = object() @@ -251,6 +270,7 @@ class PathAccessError(KeyError, IndexError, TypeError): representing what can occur when looking up a path in a nested object. """ + def __init__(self, exc, seg, path): self.exc = exc self.seg = seg @@ -258,11 +278,14 @@ class PathAccessError(KeyError, IndexError, TypeError): def __repr__(self): cn = self.__class__.__name__ - return '%s(%r, %r, %r)' % (cn, self.exc, self.seg, self.path) + return "%s(%r, %r, %r)" % (cn, self.exc, self.seg, self.path) def __str__(self): - return ('could not access %r from path %r, got error: %r' - % (self.seg, self.path, self.exc)) + return "could not access %r from path %r, got error: %r" % ( + self.seg, + self.path, + self.exc, + ) def get_path(root, path, default=_UNSET): @@ -292,7 +315,7 @@ def get_path(root, path, default=_UNSET): ``PathAccessError`` exceptions be raised. """ if isinstance(path, six.string_types): - path = path.split('.') + path = path.split(".") cur = root try: for seg in path: @@ -308,8 +331,9 @@ def get_path(root, path, default=_UNSET): cur = cur[seg] except (ValueError, KeyError, IndexError, TypeError): if not getattr(cur, "__iter__", None): - exc = TypeError('%r object is not indexable' - % type(cur).__name__) + exc = TypeError( + "%r object is not indexable" % type(cur).__name__ + ) raise PathAccessError(exc, seg, path) except PathAccessError: if default is _UNSET: @@ -373,12 +397,13 @@ def dict_path_exit(path, key, old_parent, new_parent, new_items): except AttributeError: ret = new_parent.__class__(vals) # frozensets else: - raise RuntimeError('unexpected iterable type: %r' % type(new_parent)) + raise RuntimeError("unexpected iterable type: %r" % type(new_parent)) return ret -def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, - **kwargs): +def remap( + root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, **kwargs +): """The remap ("recursive map") function is used to traverse and transform nested structures. Lists, tuples, sets, and dictionaries are just a few of the data structures nested into heterogenous @@ -462,14 +487,14 @@ def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, # TODO: improve argument formatting in sphinx doc # TODO: enter() return (False, items) to continue traverse but cancel copy? if not callable(visit): - raise TypeError('visit expected callable, not: %r' % visit) + raise TypeError("visit expected callable, not: %r" % visit) if not callable(enter): - raise TypeError('enter expected callable, not: %r' % enter) + raise TypeError("enter expected callable, not: %r" % enter) if not callable(exit): - raise TypeError('exit expected callable, not: %r' % exit) - reraise_visit = kwargs.pop('reraise_visit', True) + raise TypeError("exit expected callable, not: %r" % exit) + reraise_visit = kwargs.pop("reraise_visit", True) if kwargs: - raise TypeError('unexpected keyword arguments: %r' % kwargs.keys()) + raise TypeError("unexpected keyword arguments: %r" % kwargs.keys()) path, registry, stack = (), {}, [(None, root)] new_items_stack = [] @@ -492,8 +517,10 @@ def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, new_parent, new_items = res except TypeError: # TODO: handle False? - raise TypeError('enter should return a tuple of (new_parent,' - ' items_iterator), not: %r' % res) + raise TypeError( + "enter should return a tuple of (new_parent," + " items_iterator), not: %r" % res + ) if new_items is not False: # traverse unless False is explicitly passed registry[id_value] = new_parent @@ -524,7 +551,7 @@ def remap(root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, try: new_items_stack[-1][1].append(visited_item) except IndexError: - raise TypeError('expected remappable root, not: %r' % root) + raise TypeError("expected remappable root, not: %r" % root) return value @@ -554,14 +581,15 @@ def merge_items(target_list, sourced=False): for t_name, target in target_list: if sourced: + def remerge_visit(path, key, value): source_map[path + (key,)] = t_name return True + else: remerge_visit = default_visit - ret = remap(target, enter=remerge_enter, visit=remerge_visit, - exit=remerge_exit) + ret = remap(target, enter=remerge_enter, visit=remerge_visit, exit=remerge_exit) if not sourced: return ret