From 734b4e7759d5283a28ca77438aeb108470da39c4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 7 Oct 2018 15:41:15 -0400 Subject: [PATCH] Clean up redundant code and avoid circular imports Signed-off-by: Dan Ryan --- pipenv/core.py | 57 ++++++++++++++++++++++++------------------ pipenv/environments.py | 2 +- pipenv/project.py | 57 +++++++++++++++++++++++++----------------- pipenv/shells.py | 4 +-- pipenv/utils.py | 30 ++++++---------------- 5 files changed, 77 insertions(+), 73 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 17827ea0..3ef6abd3 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -14,6 +14,7 @@ import dotenv import delegator import pipfile from blindspin import spinner +import vistir import six from .cmdparse import Script @@ -37,11 +38,9 @@ from .utils import ( is_pinned, is_star, rmtree, - fs_str, clean_resolved_dep, - parse_indexes + parse_indexes, ) -from ._compat import TemporaryDirectory, Path from . import environments, pep508checker, progress from .environments import ( PIPENV_COLORBLIND, @@ -68,9 +67,7 @@ BAD_PACKAGES = ( "wheel", ) -FIRST_PACKAGES = ( - "cython", -) +FIRST_PACKAGES = ("cython",) # Are we using the default Python? USING_DEFAULT_PYTHON = True if not PIPENV_HIDE_EMOJIS: @@ -886,7 +883,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): cmd.append("--system-site-packages") if pypi_mirror: - pip_config = {"PIP_INDEX_URL": fs_str(pypi_mirror)} + pip_config = {"PIP_INDEX_URL": vistir.misc.fs_str(pypi_mirror)} else: pip_config = {} @@ -909,7 +906,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): # This mimics Pew's "setproject". project_file_name = os.path.join(project.virtualenv_location, ".project") with open(project_file_name, "w") as f: - f.write(fs_str(project.project_directory)) + f.write(vistir.misc.fs_str(project.project_directory)) # Say where the virtualenv is. do_where(virtualenv=True, bare=False) @@ -1196,7 +1193,9 @@ def do_init( ensure_pipfile(system=system) if not requirements_dir: cleanup_reqdir = True - requirements_dir = TemporaryDirectory(suffix="-requirements", prefix="pipenv-") + requirements_dir = vistir.compat.TemporaryDirectory( + suffix="-requirements", prefix="pipenv-" + ) # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored if (project.lockfile_exists and not ignore_pipfile) and not skip_lock: old_hash = project.get_lockfile_hash() @@ -1338,7 +1337,7 @@ def pip_install( sources = [{"url": index}] if extra_indexes: if isinstance(extra_indexes, six.string_types): - extra_indexes = [extra_indexes,] + extra_indexes = [extra_indexes] for idx in extra_indexes: try: extra_src = project.find_source(idx).get("url") @@ -1389,27 +1388,33 @@ def pip_install( if environments.is_verbose(): click.echo("$ {0}".format(pip_command), err=True) - cache_dir = Path(PIPENV_CACHE_DIR) + cache_dir = vistir.compat.Path(PIPENV_CACHE_DIR) pip_config = { - "PIP_CACHE_DIR": fs_str(cache_dir.as_posix()), - "PIP_WHEEL_DIR": fs_str(cache_dir.joinpath("wheels").as_posix()), - "PIP_DESTINATION_DIR": fs_str(cache_dir.joinpath("pkgs").as_posix()), - "PIP_EXISTS_ACTION": fs_str("w"), - "PATH": fs_str(os.environ.get("PATH")), + "PIP_CACHE_DIR": vistir.misc.fs_str(cache_dir.as_posix()), + "PIP_WHEEL_DIR": vistir.misc.fs_str(cache_dir.joinpath("wheels").as_posix()), + "PIP_DESTINATION_DIR": vistir.misc.fs_str( + cache_dir.joinpath("pkgs").as_posix() + ), + "PIP_EXISTS_ACTION": vistir.misc.fs_str("w"), + "PATH": vistir.misc.fs_str(os.environ.get("PATH")), } if src: - pip_config.update({"PIP_SRC": fs_str(project.virtualenv_src_location)}) + pip_config.update( + {"PIP_SRC": vistir.misc.fs_str(project.virtualenv_src_location)} + ) pip_command = Script.parse(pip_command).cmdify() c = delegator.run(pip_command, block=block, env=pip_config) return c def pip_download(package_name): - cache_dir = Path(PIPENV_CACHE_DIR) + cache_dir = vistir.compat.Path(PIPENV_CACHE_DIR) pip_config = { - "PIP_CACHE_DIR": fs_str(cache_dir.as_posix()), - "PIP_WHEEL_DIR": fs_str(cache_dir.joinpath("wheels").as_posix()), - "PIP_DESTINATION_DIR": fs_str(cache_dir.joinpath("pkgs").as_posix()), + "PIP_CACHE_DIR": vistir.misc.fs_str(cache_dir.as_posix()), + "PIP_WHEEL_DIR": vistir.misc.fs_str(cache_dir.joinpath("wheels").as_posix()), + "PIP_DESTINATION_DIR": vistir.misc.fs_str( + cache_dir.joinpath("pkgs").as_posix() + ), } for source in project.sources: cmd = '{0} download "{1}" -i {2} -d {3}'.format( @@ -1666,7 +1671,7 @@ def do_install( from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM from notpip._internal.exceptions import PipError - requirements_directory = TemporaryDirectory( + requirements_directory = vistir.compat.TemporaryDirectory( suffix="-requirements", prefix="pipenv-" ) if selective_upgrade: @@ -2037,7 +2042,7 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror # Ensure that virtualenv is available. ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) # Set an environment variable, so we know we're in the environment. - os.environ["PIPENV_ACTIVE"] = fs_str("1") + os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") # Support shell compatibility mode. if PIPENV_SHELL_FANCY: fancy = True @@ -2114,7 +2119,7 @@ def inline_activate_virtual_environment(): else: _inline_activate_virtualenv() if "VIRTUAL_ENV" not in os.environ: - os.environ["VIRTUAL_ENV"] = fs_str(root) + os.environ["VIRTUAL_ENV"] = vistir.misc.fs_str(root) def _launch_windows_subprocess(script): @@ -2471,7 +2476,9 @@ def do_sync( ) # Install everything. - requirements_dir = TemporaryDirectory(suffix="-requirements", prefix="pipenv-") + requirements_dir = vistir.compat.TemporaryDirectory( + suffix="-requirements", prefix="pipenv-" + ) do_init( dev=dev, concurrent=(not sequential), diff --git a/pipenv/environments.py b/pipenv/environments.py index f93f5db6..ca05535f 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -1,7 +1,7 @@ import os import sys from appdirs import user_cache_dir -from .utils import fs_str +from .vendor.vistir.misc import fs_str # HACK: avoid resolver.py uses the wrong byte code files. diff --git a/pipenv/project.py b/pipenv/project.py index b02e94d8..0722619f 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -12,14 +12,11 @@ from first import first import pipfile import pipfile.api import six +import vistir import toml -from ._compat import Path - from .cmdparse import Script from .utils import ( - atomic_open_for_write, - mkdir_p, pep423_name, proper_case, find_requirements, @@ -33,7 +30,7 @@ from .utils import ( is_star, get_workon_home, is_virtual_environment, - looks_like_dir + looks_like_dir, ) from .environments import ( PIPENV_MAX_DEPTH, @@ -50,7 +47,7 @@ from requirementslib.utils import is_vcs def _normalized(p): if p is None: return None - loc = Path(p) + loc = vistir.compat.Path(p) if loc.is_absolute(): return normalize_drive(str(loc)) else: @@ -73,13 +70,15 @@ class _LockFileEncoder(json.JSONEncoder): * PrettyTOML's container elements are seamlessly encodable. * The output is always UTF-8-encoded text, never binary, even on Python 2. """ + def __init__(self): super(_LockFileEncoder, self).__init__( - indent=4, separators=(",", ": "), sort_keys=True, + indent=4, separators=(",", ": "), sort_keys=True ) def default(self, obj): from prettytoml.elements.common import ContainerElement, TokenElement + if isinstance(obj, (ContainerElement, TokenElement)): return obj.primitive_value return super(_LockFileEncoder, self).default(obj) @@ -277,19 +276,26 @@ class Project(object): name = f.read().strip() # Assume file's contents is a path if it contains slashes. if looks_like_dir(name): - return Path(name).absolute().as_posix() + return vistir.compat.Path(name).absolute().as_posix() return str(get_workon_home().joinpath(name)) def get_installed_packages(self): from . import PIPENV_ROOT, PIPENV_VENDOR, PIPENV_PATCHED from .utils import temp_path, load_path, temp_environ + if self.virtualenv_exists: with temp_path(), temp_environ(): new_path = load_path(self.which("python")) - new_path = [new_path[0], PIPENV_ROOT, PIPENV_PATCHED, PIPENV_VENDOR] + new_path[1:] + new_path = [ + new_path[0], + PIPENV_ROOT, + PIPENV_PATCHED, + PIPENV_VENDOR, + ] + new_path[1:] sys.path = new_path - os.environ['VIRTUAL_ENV'] = self.virtualenv_location + os.environ["VIRTUAL_ENV"] = self.virtualenv_location from .vendor.pip_shims.shims import get_installed_distributions + return get_installed_distributions(local_only=True) else: return [] @@ -330,7 +336,7 @@ class Project(object): # In-project venv # "Proper" path casing (on non-case-sensitive filesystems). if ( - fnmatch.fnmatch('A', 'a') + fnmatch.fnmatch("A", "a") or self.is_venv_in_project() or get_workon_home().joinpath(venv_name).exists() ): @@ -365,7 +371,7 @@ class Project(object): if PIPENV_VIRTUALENV: return PIPENV_VIRTUALENV - if not self._virtualenv_location: # Use cached version, if available. + if not self._virtualenv_location: # Use cached version, if available. assert self.project_directory, "project not created" self._virtualenv_location = self.get_location_for_virtualenv() return self._virtualenv_location @@ -376,7 +382,7 @@ class Project(object): loc = os.sep.join([self.virtualenv_location, "src"]) else: loc = os.sep.join([self.project_directory, "src"]) - mkdir_p(loc) + vistir.path.mkdir_p(loc) return loc @property @@ -385,13 +391,13 @@ class Project(object): loc = os.sep.join([self.virtualenv_location, "downloads"]) self._download_location = loc # Create the directory, if it doesn't exist. - mkdir_p(self._download_location) + vistir.path.mkdir_p(self._download_location) return self._download_location @property def proper_names_db_path(self): if self._proper_names_db_path is None: - self._proper_names_db_path = Path( + self._proper_names_db_path = vistir.compat.Path( self.virtualenv_location, "pipenv-proper-names.txt" ) self._proper_names_db_path.touch() # Ensure the file exists. @@ -671,7 +677,10 @@ class Project(object): data[section][package].update(_data) formatted_data = toml.dumps(data).rstrip() - if Path(path).absolute() == Path(self.pipfile_location).absolute(): + if ( + vistir.compat.Path(path).absolute() + == vistir.compat.Path(self.pipfile_location).absolute() + ): newlines = self._pipfile_newlines else: newlines = DEFAULT_NEWLINES @@ -685,11 +694,10 @@ class Project(object): """Write out the lockfile. """ s = self._lockfile_encoder.encode(content) - open_kwargs = { - 'newline': self._lockfile_newlines, - 'encoding': 'utf-8', - } - with atomic_open_for_write(self.lockfile_location, **open_kwargs) as f: + open_kwargs = {"newline": self._lockfile_newlines, "encoding": "utf-8"} + with vistir.contextmanagers.atomic_open_for_write( + self.lockfile_location, **open_kwargs + ) as f: f.write(s) # Write newline at end of document. GH-319. # Only need '\n' here; the file object handles the rest. @@ -802,7 +810,9 @@ class Project(object): source = {"url": index, "verify_ssl": verify_ssl} else: return - name, _, tld_guess = six.moves.urllib.parse.urlsplit(index).netloc.rpartition(".") + name, _, tld_guess = six.moves.urllib.parse.urlsplit(index).netloc.rpartition( + "." + ) src_name = name.replace(".", "") try: self.get_source(name=src_name) @@ -810,6 +820,7 @@ class Project(object): source[name] = src_name else: from random import randint + source[name] = "{0}-{1}".format(src_name, randint(1, 1000)) # Add the package to the group. if "source" not in p: @@ -824,7 +835,7 @@ class Project(object): self.write_toml(self.parsed_pipfile) def load_lockfile(self, expand_env_vars=True): - with io.open(self.lockfile_location, encoding='utf-8') as lock: + with io.open(self.lockfile_location, encoding="utf-8") as lock: j = json.load(lock) self._lockfile_newlines = preferred_newlines(lock) # lockfile is just a string diff --git a/pipenv/shells.py b/pipenv/shells.py index 70b236ea..352c01f4 100644 --- a/pipenv/shells.py +++ b/pipenv/shells.py @@ -5,9 +5,9 @@ import signal import subprocess import sys -from ._compat import get_terminal_size, Path from .environments import PIPENV_SHELL_EXPLICIT, PIPENV_SHELL, PIPENV_EMULATOR -from .utils import temp_environ +from .vendor.vistir.compat import get_terminal_size, Path +from .vendor.vistir.contextmanagers import temp_environ from .vendor import shellingham diff --git a/pipenv/utils.py b/pipenv/utils.py index dfc33cd6..8aeb0cff 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -32,8 +32,6 @@ except ImportError: logging.basicConfig(level=logging.ERROR) -from time import time - from distutils.spawn import find_executable from contextlib import contextmanager from . import environments @@ -59,13 +57,14 @@ requests_session = None def _get_requests_session(): """Load requests lazily.""" global requests_session - from .environments import PIPENV_MAX_RETRIES if requests_session is not None: return requests_session import requests requests_session = requests.Session() - adapter = requests.adapters.HTTPAdapter(max_retries=PIPENV_MAX_RETRIES) + adapter = requests.adapters.HTTPAdapter( + max_retries=environments.PIPENV_MAX_RETRIES + ) requests_session.mount("https://pypi.org/pypi", adapter) return requests_session @@ -229,7 +228,6 @@ def actually_resolve_deps( from pipenv.patched.piptools.exceptions import NoCandidateFound from .vendor.requirementslib import Requirement from ._compat import TemporaryDirectory, NamedTemporaryFile - from .environments import PIPENV_MAX_ROUNDS, PIPENV_CACHE_DIR class PipCommand(basecommand.Command): """Needed for pip-tools.""" @@ -283,7 +281,7 @@ def actually_resolve_deps( f.write(u"\n".join([_constraint for _constraint in constraints])) constraints_file = f.name pip_options, _ = pip_command.parser.parse_args(pip_args) - pip_options.cache_dir = PIPENV_CACHE_DIR + pip_options.cache_dir = environments.PIPENV_CACHE_DIR session = pip_command._build_session(pip_options) pypi = PyPIRepository(pip_options=pip_options, use_json=False, session=session) constraints = parse_requirements( @@ -300,7 +298,7 @@ def actually_resolve_deps( # pre-resolve instead of iterating to avoid asking pypi for hashes of editable packages hashes = None try: - results = resolver.resolve(max_rounds=PIPENV_MAX_ROUNDS) + results = resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) hashes = resolver.resolve_hashes(results) resolved_tree.update(results) except (NoCandidateFound, DistributionNotFound, HTTPError) as e: @@ -344,6 +342,7 @@ def venv_resolve_deps( allow_global=False, pypi_mirror=None, ): + from .vendor.vistir.misc import fs_str from .vendor import delegator from . import resolver import json @@ -1229,20 +1228,6 @@ def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None): return {name: lockfile} -def fs_str(string): - """Encodes a string into the proper filesystem encoding - - Borrowed from pip-tools - """ - if isinstance(string, str): - return string - assert not isinstance(string, bytes) - return string.encode(_fs_encoding) - - -_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() - - def get_workon_home(): from ._compat import Path @@ -1282,7 +1267,8 @@ def is_virtual_environment(path): @contextmanager def locked_repository(requirement): - from pipenv.vendor.vistir.path import create_tracked_tempdir + from .vendor.vistir.path import create_tracked_tempdir + from .vendor.vistir.misc import fs_str src_dir = create_tracked_tempdir(prefix="pipenv-src") if not requirement.is_vcs: return