From 4bd02aa6cc11aea9c314eb15abd0cdb9cd963547 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 23 Mar 2019 17:38:20 -0400 Subject: [PATCH] Fix temporary directory cleanup Signed-off-by: Dan Ryan --- pipenv/patched/notpip/_internal/utils/misc.py | 4 +- pipenv/vendor/vistir/compat.py | 17 ++++- pipenv/vendor/vistir/path.py | 65 +++++++++++++++++-- 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/pipenv/patched/notpip/_internal/utils/misc.py b/pipenv/patched/notpip/_internal/utils/misc.py index a80153df..0a3237a2 100644 --- a/pipenv/patched/notpip/_internal/utils/misc.py +++ b/pipenv/patched/notpip/_internal/utils/misc.py @@ -117,8 +117,8 @@ def get_prog(): @retry(stop_max_delay=3000, wait_fixed=500) def rmtree(dir, ignore_errors=False): # type: (str, bool) -> None - shutil.rmtree(dir, ignore_errors=ignore_errors, - onerror=rmtree_errorhandler) + from pipenv.vendor.vistir.path import rmtree as vistir_rmtree, handle_remove_readonly + vistir_rmtree(dir, onerror=handle_remove_readonly, ignore_errors=ignore_errors) def rmtree_errorhandler(func, path, exc_info): diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index ecc6bc26..f0266e30 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -21,6 +21,8 @@ __all__ = [ "FileNotFoundError", "ResourceWarning", "PermissionError", + "is_type_checking", + "IS_TYPE_CHECKING", "IsADirectoryError", "fs_str", "lru_cache", @@ -132,8 +134,21 @@ if not sys.warnoptions: warnings.simplefilter("default", ResourceWarning) +def is_type_checking(): + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +IS_TYPE_CHECKING = is_type_checking() + + class TemporaryDirectory(object): - """Create and return a temporary directory. This has the same + + """ + Create and return a temporary directory. This has the same behavior as mkdtemp but can be used as a context manager. For example: diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 429c826e..e9370c8d 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -8,6 +8,7 @@ import os import posixpath import shutil import stat +import time import warnings import six @@ -26,8 +27,13 @@ from .compat import ( finalize, fs_decode, fs_encode, + IS_TYPE_CHECKING, ) +if IS_TYPE_CHECKING: + from typing import Optional, Callable, Text, ByteString, AnyStr + + __all__ = [ "check_for_unc_path", "get_converted_relative_path", @@ -89,6 +95,7 @@ else: def normalize_path(path): + # type: (AnyStr) -> AnyStr """ Return a case-normalized absolute variable-expanded path. @@ -105,6 +112,7 @@ def normalize_path(path): def is_in_path(path, parent): + # type: (AnyStr, AnyStr) -> bool """ Determine if the provided full path is in the given parent root. @@ -118,6 +126,7 @@ def is_in_path(path, parent): def normalize_drive(path): + # type: (str) -> Text """Normalize drive in path so they stay consistent. This currently only affects local drives on Windows, which can be @@ -138,6 +147,7 @@ def normalize_drive(path): def path_to_url(path): + # type: (str) -> Text """Convert the supplied local path to a file uri. :param str path: A string pointing to or representing a local path @@ -157,7 +167,9 @@ def path_to_url(path): def url_to_path(url): - """Convert a valid file url to a local filesystem path + # type: (str) -> ByteString + """ + Convert a valid file url to a local filesystem path Follows logic taken from pip's equivalent function """ @@ -296,10 +308,13 @@ def create_tracked_tempfile(*args, **kwargs): def set_write_bit(fn): - """Set read-write permissions for the current user on the target path. Fail silently + # type: (str) -> None + """ + Set read-write permissions for the current user on the target path. Fail silently if the path doesn't exist. :param str fn: The target filename or path + :return: None """ fn = fs_encode(fn) @@ -313,6 +328,7 @@ def set_write_bit(fn): os.chflags(path, 0) except AttributeError: pass + return None for root, dirs, files in os.walk(fn, topdown=False): for dir_ in [os.path.join(root, d) for d in dirs]: set_write_bit(dir_) @@ -321,7 +337,9 @@ def set_write_bit(fn): def rmtree(directory, ignore_errors=False, onerror=None): - """Stand-in for :func:`~shutil.rmtree` with additional error-handling. + # type: (str, bool, Optional[Callable]) -> None + """ + Stand-in for :func:`~shutil.rmtree` with additional error-handling. This version of `rmtree` handles read-only paths, especially in the case of index files written by certain source control systems. @@ -346,6 +364,39 @@ def rmtree(directory, ignore_errors=False, onerror=None): raise +def _wait_for_files(path): + """ + Retry with backoff up to 1 second to delete files from a directory. + + :param str path: The path to crawl to delete files from + :return: A list of remaining paths or None + :rtype: Optional[List[str]] + """ + timeout = 0.001 + remaining = [] + while timeout < 1.0: + remaining = [] + if os.path.isdir(path): + L = os.listdir(path) + for target in L: + _remaining = _wait_for_files(target) + if _remaining: + remaining.extend(_remaining) + continue + try: + os.unlink(path) + except FileNotFoundError as e: + if e.errno == errno.ENOENT: + return + except (OSError, IOError, PermissionError): + time.sleep(timeout) + timeout *= 2 + remaining.append(path) + else: + return + return remaining + + def handle_remove_readonly(func, path, exc): """Error handler for shutil.rmtree. @@ -375,11 +426,17 @@ def handle_remove_readonly(func, path, exc): if e.errno == errno.ENOENT: return elif e.errno in PERM_ERRORS: - warnings.warn(default_warning_message.format(path), ResourceWarning) + remaining = None + if os.path.isdir(path): + remaining =_wait_for_files(path) + if remaining: + warnings.warn(default_warning_message.format(path), ResourceWarning) return + raise if exc_exception.errno in PERM_ERRORS: set_write_bit(path) + remaining = _wait_for_files(path) try: func(path) except (OSError, IOError, FileNotFoundError, PermissionError) as e: