Update pythonfinder, requirementslib and vistir

Signed-off-by: Dan Ryan <dan@danryan.co>
This commit is contained in:
Dan Ryan
2019-05-11 01:30:25 -04:00
parent 1006b50bcd
commit 90c2c66dc8
16 changed files with 609 additions and 272 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ from .exceptions import InvalidPythonVersion
from .models import SystemPath, WindowsFinder
from .pythonfinder import Finder
__version__ = "1.2.0"
__version__ = "1.2.1"
logger = logging.getLogger(__name__)
+35 -36
View File
@@ -12,6 +12,7 @@ import attr
import six
from cached_property import cached_property
from vistir.compat import Path, fs_str
from vistir.misc import dedup
from .mixins import BaseFinder, BasePath
from .python import PythonVersion
@@ -38,6 +39,7 @@ from ..utils import (
parse_asdf_version_order,
parse_pyenv_version_order,
path_is_known_executable,
split_version_and_name,
unnest,
)
@@ -209,6 +211,7 @@ class SystemPath(object):
path_entries = self.paths.copy()
if self.global_search and "PATH" in os.environ:
path_order = path_order + os.environ["PATH"].split(os.pathsep)
path_order = list(dedup(path_order))
path_instances = [
ensure_path(p.strip('"'))
for p in path_order
@@ -439,7 +442,7 @@ class SystemPath(object):
return _path
def _get_paths(self):
# type: () -> Generator[PathType, None, None]
# type: () -> Generator[Union[PathType, WindowsFinder], None, None]
for path in self.path_order:
try:
entry = self.get_path(path)
@@ -450,7 +453,7 @@ class SystemPath(object):
@cached_property
def path_entries(self):
# type: () -> List[Union[PathEntry, FinderType]]
# type: () -> List[Union[PathType, WindowsFinder]]
paths = list(self._get_paths())
return paths
@@ -558,6 +561,7 @@ class SystemPath(object):
dev=None, # type: Optional[bool]
arch=None, # type: Optional[str]
name=None, # type: Optional[str]
sort_by_path=False, # type: bool
):
# type: (...) -> PathEntry
"""Search for a specific python version on the path.
@@ -570,29 +574,12 @@ class SystemPath(object):
:param bool dev: Search for devreleases (default None) - prioritize releases if None
:param str arch: Architecture to include, e.g. '64bit', defaults to None
:param str name: The name of a python version, e.g. ``anaconda3-5.3.0``
:param bool sort_by_path: Whether to sort by path -- default sort is by version(default: False)
:return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested.
:rtype: :class:`~pythonfinder.models.PathEntry`
"""
if isinstance(major, six.string_types) and not minor and not patch:
# Only proceed if this is in the format "x.y.z" or similar
if major.isdigit() or (major.count(".") > 0 and major[0].isdigit()):
version = major.split(".", 2)
if isinstance(version, (tuple, list)):
if len(version) > 3:
major, minor, patch, rest = version
elif len(version) == 3:
major, minor, patch = version
elif len(version) == 2:
major, minor = version
else:
major = major[0]
else:
major = major
name = None
else:
name = "{0!s}".format(major)
major = None
major, minor, patch, name = split_version_and_name(major, minor, patch, name)
sub_finder = operator.methodcaller(
"find_python_version", major, minor, patch, pre, dev, arch, name
)
@@ -610,6 +597,18 @@ class SystemPath(object):
windows_finder_version = sub_finder(self.windows_finder)
if windows_finder_version:
return windows_finder_version
if sort_by_path:
paths = [self.get_path(k) for k in self.path_order]
for path in paths:
found_version = sub_finder(path)
if found_version:
return found_version
if alternate_sub_finder:
for path in paths:
found_version = alternate_sub_finder(path)
if found_version:
return found_version
ver = next(iter(self.get_pythons(sub_finder)), None)
if not ver and alternate_sub_finder is not None:
ver = next(iter(self.get_pythons(alternate_sub_finder)), None)
@@ -647,9 +646,9 @@ class SystemPath(object):
paths = [] # type: List[str]
if ignore_unsupported:
os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1")
# if global_search:
# if "PATH" in os.environ:
# paths = os.environ["PATH"].split(os.pathsep)
if global_search:
if "PATH" in os.environ:
paths = os.environ["PATH"].split(os.pathsep)
path_order = []
if path:
path_order = [path]
@@ -663,18 +662,18 @@ class SystemPath(object):
)
}
)
# paths = [path] + paths
# paths = [p for p in paths if not any(is_in_path(p, shim) for shim in SHIM_PATHS)]
# _path_objects = [ensure_path(p.strip('"')) for p in paths]
# paths = [p.as_posix() for p in _path_objects]
# path_entries.update(
# {
# p.as_posix(): PathEntry.create(
# path=p.absolute(), is_root=True, only_python=only_python
# )
# for p in _path_objects
# }
# )
paths = [path] + paths
paths = [p for p in paths if not any(is_in_path(p, shim) for shim in SHIM_PATHS)]
_path_objects = [ensure_path(p.strip('"')) for p in paths]
paths = [p.as_posix() for p in _path_objects]
path_entries.update(
{
p.as_posix(): PathEntry.create(
path=p.absolute(), is_root=True, only_python=only_python
)
for p in _path_objects
}
)
instance = cls(
paths=path_entries,
path_order=path_order,
+120 -60
View File
@@ -14,11 +14,13 @@ from .exceptions import InvalidPythonVersion
from .utils import Iterable, filter_pythons, version_re
if environment.MYPY_RUNNING:
from typing import Optional, Dict, Any, Union, List, Iterator
from typing import Optional, Dict, Any, Union, List, Iterator, Text
from .models.path import Path, PathEntry
from .models.windows import WindowsFinder
from .models.path import SystemPath
STRING_TYPE = Union[str, Text, bytes]
class Finder(object):
@@ -33,9 +35,14 @@ class Finder(object):
"""
def __init__(
self, path=None, system=False, global_search=True, ignore_unsupported=True
self,
path=None,
system=False,
global_search=True,
ignore_unsupported=True,
sort_by_path=False,
):
# type: (Optional[str], bool, bool, bool) -> None
# type: (Optional[str], bool, bool, bool, bool) -> None
"""Create a new :class:`~pythonfinder.pythonfinder.Finder` instance.
:param path: A bin-directory search location, defaults to None
@@ -46,12 +53,14 @@ class Finder(object):
:param global_search: bool, optional
:param ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True
:param ignore_unsupported: bool, optional
:param bool sort_by_path: Whether to always sort by path
:returns: a :class:`~pythonfinder.pythonfinder.Finder` object.
"""
self.path_prepend = path # type: Optional[str]
self.global_search = global_search # type: bool
self.system = system # type: bool
self.sort_by_path = sort_by_path # type: bool
self.ignore_unsupported = ignore_unsupported # type: bool
self._system_path = None # type: Optional[SystemPath]
self._windows_finder = None # type: Optional[WindowsFinder]
@@ -92,7 +101,7 @@ class Finder(object):
self._system_path = self.create_system_path()
def rehash(self):
# type: () -> None
# type: () -> "Finder"
if not self._system_path:
self._system_path = self.create_system_path()
self.find_all_python_versions.cache_clear()
@@ -123,11 +132,92 @@ class Finder(object):
# type: (str) -> Optional[PathEntry]
return self.system_path.which(exe)
@classmethod
def parse_major(cls, major, minor=None, patch=None, pre=None, dev=None, arch=None):
# type: (Optional[str], Optional[int], Optional[int], Optional[bool], Optional[bool], Optional[str]) -> Dict[str, Union[int, str, bool, None]]
from .models import PythonVersion
major_is_str = major and isinstance(major, six.string_types)
is_num = (
major
and major_is_str
and all(part.isdigit() for part in major.split(".")[:2])
)
major_has_arch = (
arch is None
and major
and major_is_str
and "-" in major
and major[0].isdigit()
)
name = None
if major and major_has_arch:
orig_string = "{0!s}".format(major)
major, _, arch = major.rpartition("-")
if arch:
arch = arch.lower().lstrip("x").replace("bit", "")
if not (arch.isdigit() and (int(arch) & int(arch) - 1) == 0):
major = orig_string
arch = None
else:
arch = "{0}bit".format(arch)
try:
version_dict = PythonVersion.parse(major)
except (ValueError, InvalidPythonVersion):
if name is None:
name = "{0!s}".format(major)
major = None
version_dict = {}
elif major and major[0].isalpha():
return {"major": None, "name": major, "arch": arch}
elif major and is_num:
match = version_re.match(major)
version_dict = match.groupdict() if match else {} # type: ignore
version_dict.update(
{
"is_prerelease": bool(version_dict.get("prerel", False)),
"is_devrelease": bool(version_dict.get("dev", False)),
}
)
else:
version_dict = {
"major": major,
"minor": minor,
"patch": patch,
"pre": pre,
"dev": dev,
"arch": arch,
}
if not version_dict.get("arch") and arch:
version_dict["arch"] = arch
version_dict["minor"] = (
int(version_dict["minor"]) if version_dict.get("minor") is not None else minor
)
version_dict["patch"] = (
int(version_dict["patch"]) if version_dict.get("patch") is not None else patch
)
version_dict["major"] = (
int(version_dict["major"]) if version_dict.get("major") is not None else major
)
if not (version_dict["major"] or version_dict.get("name")):
version_dict["major"] = major
if name:
version_dict["name"] = name
return version_dict
@lru_cache(maxsize=1024)
def find_python_version(
self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None
self,
major=None, # type: Optional[Union[str, int]]
minor=None, # type: Optional[int]
patch=None, # type: Optional[int]
pre=None, # type: Optional[bool]
dev=None, # type: Optional[bool]
arch=None, # type: Optional[str]
name=None, # type: Optional[str]
sort_by_path=False, # type: bool
):
# type: (Optional[Union[str, int]], Optional[int], Optional[int], Optional[bool], Optional[bool], Optional[str], Optional[str]) -> PathEntry
# type: (...) -> Optional[PathEntry]
"""
Find the python version which corresponds most closely to the version requested.
@@ -138,18 +228,19 @@ class Finder(object):
:param Optional[bool] dev: If provided, whether to search dev-releases.
:param Optional[str] arch: If provided, which architecture to search.
:param Optional[str] name: *Name* of the target python, e.g. ``anaconda3-5.3.0``
:param bool sort_by_path: Whether to sort by path -- default sort is by version(default: False)
:return: A new *PathEntry* pointer at a matching python version, if one can be located.
:rtype: :class:`pythonfinder.models.path.PathEntry`
"""
from .models import PythonVersion
minor = int(minor) if minor is not None else minor
patch = int(patch) if patch is not None else patch
version_dict = {
"minor": minor,
"patch": patch,
"name": name,
"arch": arch,
} # type: Dict[str, Union[str, int, Any]]
if (
@@ -159,60 +250,22 @@ class Finder(object):
and dev is None
and patch is None
):
if arch is None and "-" in major and major[0].isdigit():
orig_string = "{0!s}".format(major)
major, _, arch = major.rpartition("-")
if arch.startswith("x"):
arch = arch.lstrip("x")
if arch.lower().endswith("bit"):
arch = arch.lower().replace("bit", "")
if not (arch.isdigit() and (int(arch) & int(arch) - 1) == 0):
major = orig_string
arch = None
else:
arch = "{0}bit".format(arch)
try:
version_dict = PythonVersion.parse(major)
except (ValueError, InvalidPythonVersion):
if name is None:
name = "{0!s}".format(major)
major = None
version_dict = {}
elif major[0].isalpha():
name = "%s" % major
major = None
else:
if "." in major and all(part.isdigit() for part in major.split(".")[:2]):
match = version_re.match(major)
version_dict = match.groupdict()
version_dict["is_prerelease"] = bool(
version_dict.get("prerel", False)
)
version_dict["is_devrelease"] = bool(version_dict.get("dev", False))
else:
version_dict = {
"major": major,
"minor": minor,
"patch": patch,
"pre": pre,
"dev": dev,
"arch": arch,
}
if version_dict.get("minor") is not None:
minor = int(version_dict["minor"])
if version_dict.get("patch") is not None:
patch = int(version_dict["patch"])
if version_dict.get("major") is not None:
major = int(version_dict["major"])
version_dict = self.parse_major(major, minor=minor, patch=patch, arch=arch)
major = version_dict["major"]
minor = version_dict.get("minor", minor) # type: ignore
patch = version_dict.get("patch", patch) # type: ignore
arch = version_dict.get("arch", arch) # type: ignore
name = version_dict.get("name", name) # type: ignore
_pre = version_dict.get("is_prerelease", pre)
pre = bool(_pre) if _pre is not None else pre
_dev = version_dict.get("is_devrelease", dev)
dev = bool(_dev) if _dev is not None else dev
arch = (
version_dict.get("architecture", None) if arch is None else arch
) # type: ignore
if "architecture" in version_dict and isinstance(
version_dict["architecture"], six.string_types
):
arch = version_dict["architecture"] # type: ignore
if os.name == "nt" and self.windows_finder is not None:
match = self.windows_finder.find_python_version(
found = self.windows_finder.find_python_version(
major=major,
minor=minor,
patch=patch,
@@ -221,10 +274,17 @@ class Finder(object):
arch=arch,
name=name,
)
if match:
return match
if found:
return found
return self.system_path.find_python_version(
major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name
major=major,
minor=minor,
patch=patch,
pre=pre,
dev=dev,
arch=arch,
name=name,
sort_by_path=self.sort_by_path,
)
@lru_cache(maxsize=1024)
+30 -1
View File
@@ -104,7 +104,7 @@ def get_python_version(path):
combine_stderr=False,
write_to_stdout=False,
)
timer = Timer(5, c.kill)
timer = Timer(SUBPROCESS_TIMEOUT, c.kill)
except OSError:
raise InvalidPythonVersion("%s is not a valid python path" % path)
if not c.out:
@@ -334,6 +334,35 @@ def parse_asdf_version_order(filename=".tool-versions"):
return []
def split_version_and_name(
major=None, # type: Optional[Union[str, int]]
minor=None, # type: Optional[Union[str, int]]
patch=None, # type: Optional[Union[str, int]]
name=None, # type: Optional[str]
):
# type: (...) -> Tuple[Optional[Union[str, int]], Optional[Union[str, int]], Optional[Union[str, int]], Optional[str]]
if isinstance(major, six.string_types) and not minor and not patch:
# Only proceed if this is in the format "x.y.z" or similar
if major.isdigit() or (major.count(".") > 0 and major[0].isdigit()):
version = major.split(".", 2)
if isinstance(version, (tuple, list)):
if len(version) > 3:
major, minor, patch, _ = version
elif len(version) == 3:
major, minor, patch = version
elif len(version) == 2:
major, minor = version
else:
major = major[0]
else:
major = major
name = None
else:
name = "{0!s}".format(major)
major = None
return (major, minor, patch, name)
# TODO: Reimplement in vistir
def is_in_path(path, parent):
return normalize_path(str(path)).startswith(normalize_path(str(parent)))
+1 -1
View File
@@ -10,7 +10,7 @@ from .models.lockfile import Lockfile
from .models.pipfile import Pipfile
from .models.requirements import Requirement
__version__ = "1.4.3.dev0"
__version__ = "1.4.3"
logger = logging.getLogger(__name__)
+13 -6
View File
@@ -9,7 +9,7 @@ import six
from packaging.markers import InvalidMarker, Marker
from packaging.specifiers import Specifier, SpecifierSet
from vistir.compat import Mapping, Set, lru_cache
from vistir.misc import _is_iterable, dedup
from vistir.misc import dedup
from .utils import filter_none, validate_markers
from ..environment import MYPY_RUNNING
@@ -19,14 +19,14 @@ from six.moves import reduce # isort:skip
if MYPY_RUNNING:
from typing import Optional, List, Generic, Type
from typing import Optional, List, Type, Any
MAX_VERSIONS = {2: 7, 3: 10}
def is_instance(item, cls):
# type: (Generic, Type) -> bool
# type: (Any, Type) -> bool
if isinstance(item, cls) or item.__class__.__name__ == cls.__name__:
return True
return False
@@ -139,8 +139,12 @@ def _format_pyspec(specifier):
if not any(op in specifier for op in Specifier._operators.keys()):
specifier = "=={0}".format(specifier)
specifier = Specifier(specifier)
version = specifier.version.replace(".*", "")
if ".*" in specifier.version:
version = getattr(specifier, "version", specifier).rstrip()
if version and version.endswith("*"):
if version.endswith(".*"):
version = version.rstrip(".*")
else:
version = version.rstrip("*")
specifier = Specifier("{0}{1}".format(specifier.operator, version))
try:
op = REPLACE_RANGES[specifier.operator]
@@ -198,7 +202,10 @@ def _group_by_op(specs):
@lru_cache(maxsize=128)
def cleanup_pyspecs(specs, joiner="or"):
specs = {_format_pyspec(spec) for spec in specs}
if isinstance(specs, six.string_types):
specs = set([_format_pyspec(specs)])
else:
specs = {_format_pyspec(spec) for spec in specs}
# for != operator we want to group by version
# if all are consecutive, join as a list
results = set()
-1
View File
@@ -4,7 +4,6 @@ from __future__ import absolute_import, print_function
import collections
import copy
import hashlib
import os
import sys
from contextlib import contextmanager
+99 -85
View File
@@ -23,15 +23,7 @@ from distlib.wheel import Wheel
from packaging.markers import Marker
from six.moves import configparser
from six.moves.urllib.parse import unquote, urlparse, urlunparse
from vistir.compat import (
FileNotFoundError,
Iterable,
Mapping,
Path,
fs_decode,
fs_encode,
lru_cache,
)
from vistir.compat import FileNotFoundError, Iterable, Mapping, Path, lru_cache
from vistir.contextmanagers import cd, temp_path
from vistir.misc import run
from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p, rmtree
@@ -151,7 +143,7 @@ class HookCaller(pep517.wrappers.Pep517HookCaller):
def parse_special_directives(setup_entry, package_dir=None):
# type: (S, Optional[S]) -> S
# type: (S, Optional[STRING_TYPE]) -> S
rv = setup_entry
if not package_dir:
package_dir = os.getcwd()
@@ -209,71 +201,92 @@ def setuptools_parse_setup_cfg(path):
return results
def parse_setup_cfg(setup_cfg_path):
# type: (S) -> Dict[S, Union[S, None, Set[BaseRequirement], List[S], Tuple[S, Tuple[BaseRequirement]]]]
if os.path.exists(setup_cfg_path):
try:
return setuptools_parse_setup_cfg(setup_cfg_path)
except Exception:
pass
default_opts = {
"metadata": {"name": "", "version": ""},
"options": {
"install_requires": "",
"python_requires": "",
"build_requires": "",
"setup_requires": "",
"extras": "",
"packages.find": {"where": "."},
},
}
parser = configparser.ConfigParser(default_opts)
parser.read(setup_cfg_path)
results = {}
def get_package_dir_from_setupcfg(parser, base_dir=None):
# type: (configparser.ConfigParser, STRING_TYPE) -> Text
if not base_dir:
package_dir = os.getcwd()
if parser.has_option("options", "packages.find"):
pkg_dir = parser.get("options", "packages.find")
if isinstance(package_dir, Mapping):
package_dir = os.path.join(package_dir, pkg_dir.get("where"))
elif parser.has_option("options", "packages"):
pkg_dir = parser.get("options", "packages")
if "find:" in pkg_dir:
_, pkg_dir = pkg_dir.split("find:")
pkg_dir = pkg_dir.strip()
package_dir = os.path.join(package_dir, pkg_dir)
if parser.has_option("metadata", "name"):
results["name"] = parse_special_directives(
parser.get("metadata", "name"), package_dir
)
if parser.has_option("metadata", "version"):
results["version"] = parse_special_directives(
parser.get("metadata", "version"), package_dir
)
install_requires = set() # type: Set[BaseRequirement]
if parser.has_option("options", "install_requires"):
install_requires = make_base_requirements(
parser.get("options", "install_requires").split("\n")
)
results["install_requires"] = install_requires
if parser.has_option("options", "python_requires"):
results["python_requires"] = parse_special_directives(
parser.get("options", "python_requires"), package_dir
)
if parser.has_option("options", "build_requires"):
results["build_requires"] = parser.get("options", "build_requires")
extras = {}
if "options.extras_require" in parser.sections():
extras_require_section = parser.options("options.extras_require")
for section in extras_require_section:
if section in ["options", "metadata"]:
continue
section_contents = parser.get("options.extras_require", section)
section_list = section_contents.split("\n")
section_extras = tuple(make_base_requirements(section_list))
if section_extras:
extras[section] = section_extras
results["extras_require"] = extras
return results
else:
package_dir = base_dir
if parser.has_option("options", "packages.find"):
pkg_dir = parser.get("options", "packages.find")
if isinstance(package_dir, Mapping):
package_dir = os.path.join(package_dir, pkg_dir.get("where"))
elif parser.has_option("options", "packages"):
pkg_dir = parser.get("options", "packages")
if "find:" in pkg_dir:
_, pkg_dir = pkg_dir.split("find:")
pkg_dir = pkg_dir.strip()
package_dir = os.path.join(package_dir, pkg_dir)
return package_dir
def get_name_and_version_from_setupcfg(parser, package_dir):
# type: (configparser.ConfigParser, STRING_TYPE) -> Tuple[Optional[S], Optional[S]]
name, version = None, None
if parser.has_option("metadata", "name"):
name = parse_special_directives(parser.get("metadata", "name"), package_dir)
if parser.has_option("metadata", "version"):
version = parse_special_directives(parser.get("metadata", "version"), package_dir)
return name, version
def get_extras_from_setupcfg(parser):
# type: (configparser.ConfigParser) -> Dict[STRING_TYPE, Tuple[BaseRequirement, ...]]
extras = {} # type: Dict[STRING_TYPE, Tuple[BaseRequirement, ...]]
if "options.extras_require" not in parser.sections():
return extras
extras_require_section = parser.options("options.extras_require")
for section in extras_require_section:
if section in ["options", "metadata"]:
continue
section_contents = parser.get("options.extras_require", section)
section_list = section_contents.split("\n")
section_extras = tuple(make_base_requirements(section_list))
if section_extras:
extras[section] = section_extras
return extras
def parse_setup_cfg(setup_cfg_path):
# type: (S) -> Dict[S, Union[S, None, Set[BaseRequirement], List[S], Dict[STRING_TYPE, Tuple[BaseRequirement]]]]
if not os.path.exists(setup_cfg_path):
raise FileNotFoundError(setup_cfg_path)
try:
return setuptools_parse_setup_cfg(setup_cfg_path)
except Exception:
pass
default_opts = {
"metadata": {"name": "", "version": ""},
"options": {
"install_requires": "",
"python_requires": "",
"build_requires": "",
"setup_requires": "",
"extras": "",
"packages.find": {"where": "."},
},
}
parser = configparser.ConfigParser(default_opts)
parser.read(setup_cfg_path)
results = {}
package_dir = get_package_dir_from_setupcfg(parser, base_dir=os.getcwd())
name, version = get_name_and_version_from_setupcfg(parser, package_dir)
results["name"] = name
results["version"] = version
install_requires = set() # type: Set[BaseRequirement]
if parser.has_option("options", "install_requires"):
install_requires = make_base_requirements(
parser.get("options", "install_requires").split("\n")
)
results["install_requires"] = install_requires
if parser.has_option("options", "python_requires"):
results["python_requires"] = parse_special_directives(
parser.get("options", "python_requires"), package_dir
)
if parser.has_option("options", "build_requires"):
results["build_requires"] = parser.get("options", "build_requires")
results["extras_require"] = get_extras_from_setupcfg(parser)
return results
@contextlib.contextmanager
@@ -526,8 +539,10 @@ def get_metadata_from_wheel(wheel_path):
name = metadata.name
version = metadata.version
requires = []
extras_keys = getattr(metadata, "extras", [])
extras = {k: [] for k in extras_keys}
extras_keys = getattr(metadata, "extras", []) # type: List[STRING_TYPE]
extras = {
k: [] for k in extras_keys
} # type: Dict[STRING_TYPE, List[RequirementType]]
for req in getattr(metadata, "run_requires", []):
parsed_req = init_requirement(req)
parsed_marker = parsed_req.marker
@@ -555,7 +570,7 @@ def get_metadata_from_dist(dist):
dep_map = dist._build_dep_map()
except Exception:
dep_map = {}
deps = []
deps = [] # type: List[PkgResourcesRequirement]
extras = {}
for k in dep_map.keys():
if k is None:
@@ -573,12 +588,14 @@ def get_metadata_from_dist(dist):
else:
marker = ""
extra = "{0}".format(k)
_deps = ["{0}{1}".format(str(req), marker) for req in _deps]
_deps = ensure_reqs(tuple(_deps))
_deps = ensure_reqs(
tuple(["{0}{1}".format(str(req), marker) for req in _deps])
)
if extra:
extras[extra] = _deps
else:
deps.extend(_deps)
requires.extend(deps)
return {
"name": dist.project_name,
"version": dist.version,
@@ -634,6 +651,7 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no
unparsed = unparse(item.value)
elif isinstance(item, ast.Name):
if not initial_mapping:
unparsed = item.id
if analyzer and recurse:
if item in analyzer.assignments:
items = unparse(analyzer.assignments[item])
@@ -643,10 +661,6 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no
if assignment is not None:
items = unparse(analyzer.assignments[assignment])
unparsed = items.get(item.id, item.id)
else:
unparsed = item.id
else:
unparsed = item.id
else:
unparsed = item
elif six.PY3 and isinstance(item, ast.NameConstant):
@@ -915,7 +929,7 @@ class SetupInfo(object):
base = Path(self.extra_kwargs["src_dir"])
egg_base = base.joinpath("reqlib-metadata")
if not egg_base.exists():
atexit.register(rmtree, fs_encode(egg_base.as_posix()))
atexit.register(rmtree, egg_base.as_posix())
egg_base.mkdir(parents=True, exist_ok=True)
return egg_base.as_posix()
@@ -1160,7 +1174,7 @@ build-backend = "{1}"
metadata = [
get_metadata(d, pkg_name=self.name, metadata_type=metadata_type)
for d in metadata_dirs
if os.path.exists(fs_encode(d))
if os.path.exists(d)
]
metadata = next(iter(d for d in metadata if d), None)
return metadata
+3 -3
View File
@@ -27,7 +27,7 @@ from vistir.misc import dedup
from vistir.path import is_valid_url
from ..environment import MYPY_RUNNING
from ..utils import SCHEME_LIST, VCS_LIST, add_ssh_scheme_to_git_uri, is_star
from ..utils import SCHEME_LIST, VCS_LIST, is_star
if MYPY_RUNNING:
from typing import (
@@ -43,9 +43,9 @@ if MYPY_RUNNING:
Text,
AnyStr,
Match,
Iterable,
Iterable, # noqa
)
from attr import _ValidatorType
from attr import _ValidatorType # noqa
from packaging.requirements import Requirement as PackagingRequirement
from pkg_resources import Requirement as PkgResourcesRequirement
from pkg_resources.extern.packaging.markers import (
+15 -26
View File
@@ -1,51 +1,40 @@
# -*- coding=utf-8 -*-
from __future__ import absolute_import, print_function
import contextlib
import logging
import os
import sys
import pip_shims.shims
import six
import six.moves
import tomlkit
import vistir
from six.moves.urllib.parse import urlparse, urlsplit, urlunparse
from vistir.compat import Path
from vistir.path import create_tracked_tempdir, ensure_mkdir_p, is_valid_url
from vistir.path import ensure_mkdir_p, is_valid_url
from .environment import MYPY_RUNNING
# fmt: off
six.add_move(
six.MovedAttribute("Mapping", "collections", "collections.abc")
) # type: ignore # noqa # isort:skip
six.add_move(
six.MovedAttribute("Sequence", "collections", "collections.abc")
) # type: ignore # noqa # isort:skip
six.add_move(
six.MovedAttribute("Set", "collections", "collections.abc")
) # type: ignore # noqa # isort:skip
six.add_move(
six.MovedAttribute("ItemsView", "collections", "collections.abc")
) # type: ignore # noqa
six.add_move( # type: ignore
six.MovedAttribute("Mapping", "collections", "collections.abc") # type: ignore
) # noqa # isort:skip
six.add_move( # type: ignore
six.MovedAttribute("Sequence", "collections", "collections.abc") # type: ignore
) # noqa # isort:skip
six.add_move( # type: ignore
six.MovedAttribute("Set", "collections", "collections.abc") # type: ignore
) # noqa # isort:skip
six.add_move( # type: ignore
six.MovedAttribute("ItemsView", "collections", "collections.abc") # type: ignore
) # noqa
from six.moves import ItemsView, Mapping, Sequence, Set # type: ignore # noqa # isort:skip
# fmt: on
if MYPY_RUNNING:
from typing import (
Dict,
Any,
Optional,
Union,
Tuple,
List,
Iterable,
Generator,
Text,
TypeVar,
)
from typing import Dict, Any, Optional, Union, Tuple, List, Iterable, Text, TypeVar
STRING_TYPE = Union[bytes, str, Text]
S = TypeVar("S", bytes, str, Text)
+1 -1
View File
@@ -36,7 +36,7 @@ from .misc import (
from .path import create_tracked_tempdir, create_tracked_tempfile, mkdir_p, rmtree
from .spin import create_spinner
__version__ = "0.4.0"
__version__ = "0.4.1"
__all__ = [
+110 -1
View File
@@ -61,8 +61,9 @@ from ctypes import (
WINFUNCTYPE,
)
from ctypes.wintypes import LPWSTR, LPCWSTR
from itertools import count
from six import PY2, text_type
from .misc import StreamWrapper
from .misc import StreamWrapper, run
try:
from ctypes import pythonapi
@@ -391,3 +392,111 @@ def show_cursor():
def get_stream_handle(stream):
return STREAM_MAP.get(stream.fileno())
def _walk_for_powershell(directory):
for path, dirs, files in os.walk(directory):
powershell = next(
iter(fn for fn in files if fn.lower() == "powershell.exe"), None
)
if powershell is not None:
return os.path.join(directory, powershell)
for subdir in dirs:
powershell = _walk_for_powershell(os.path.join(directory, subdir))
if powershell:
return powershell
return None
def _get_powershell_path():
paths = [
os.path.expandvars(r"%windir%\{0}\WindowsPowerShell").format(subdir)
for subdir in ("SysWOW64", "system32")
]
powershell_path = next(iter(_walk_for_powershell(pth) for pth in paths), None)
if not powershell_path:
powershell_path, _ = run(
["where", "powershell"], block=True, nospin=True, return_object=False
)
if powershell_path:
return powershell_path.strip()
return None
def _get_sid_with_powershell():
powershell_path = _get_powershell_path()
if not powershell_path:
return None
args = [
powershell_path,
"-ExecutionPolicy",
"Bypass",
"-Command",
"Invoke-Expression '[System.Security.Principal.WindowsIdentity]::GetCurrent().user | Write-Host'",
]
sid, _ = run(args, nospin=True)
return sid.strip()
def _get_sid_from_registry():
try:
import winreg
except ImportError:
import _winreg as winreg
var_names = ("%USERPROFILE%", "%HOME%")
current_user_home = next(iter(os.path.expandvars(v) for v in var_names if v), None)
root, subkey = (
winreg.HKEY_LOCAL_MACHINE,
r"Software\Microsoft\Windows NT\CurrentVersion\ProfileList",
)
subkey_names = []
value = None
matching_key = None
try:
with winreg.OpenKeyEx(root, subkey, 0, winreg.KEY_READ) as key:
for i in count():
key_name = winreg.EnumKey(key, i)
subkey_names.append(key_name)
value = query_registry_value(
root, r"{0}\{1}".format(subkey, key_name), "ProfileImagePath"
)
if value and value.lower() == current_user_home.lower():
matching_key = key_name
break
except OSError:
pass
if matching_key is not None:
return matching_key
def get_value_from_tuple(value, value_type):
try:
import winreg
except ImportError:
import _winreg as winreg
if value_type in (winreg.REG_SZ, winreg.REG_EXPAND_SZ):
if "\0" in value:
return value[: value.index("\0")]
return value
return None
def query_registry_value(root, key_name, value):
try:
import winreg
except ImportError:
import _winreg as winreg
try:
with winreg.OpenKeyEx(root, key_name, 0, winreg.KEY_READ) as key:
return get_value_from_tuple(*winreg.QueryValueEx(key, value))
except OSError:
return None
def get_current_user():
fns = (_get_sid_from_registry, _get_sid_with_powershell)
for fn in fns:
result = fn()
if result:
return result
return None
+110 -19
View File
@@ -43,7 +43,7 @@ __all__ = [
if sys.version_info >= (3, 5):
from pathlib import Path
else:
from pipenv.vendor.pathlib2 import Path
from pathlib2 import Path
if six.PY3:
# Only Python 3.4+ is supported
@@ -53,12 +53,14 @@ if six.PY3:
from weakref import finalize
else:
# Only Python 2.7 is supported
from pipenv.vendor.backports.functools_lru_cache import lru_cache
from backports.functools_lru_cache import lru_cache
from .backports.functools import partialmethod # type: ignore
from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size
from backports.shutil_get_terminal_size import get_terminal_size
from .backports.surrogateescape import register_surrogateescape
register_surrogateescape()
NamedTemporaryFile = _NamedTemporaryFile
from pipenv.vendor.backports.weakref import finalize # type: ignore
from backports.weakref import finalize # type: ignore
try:
# Introduced Python 3.5
@@ -245,6 +247,72 @@ def _get_path(path):
return
# copied from the os backport which in turn copied this from
# the pyutf8 package --
# URL: https://github.com/etrepum/pyutf8/blob/master/pyutf8/ref.py
#
def _invalid_utf8_indexes(bytes):
skips = []
i = 0
len_bytes = len(bytes)
while i < len_bytes:
c1 = bytes[i]
if c1 < 0x80:
# U+0000 - U+007F - 7 bits
i += 1
continue
try:
c2 = bytes[i + 1]
if (c1 & 0xE0 == 0xC0) and (c2 & 0xC0 == 0x80):
# U+0080 - U+07FF - 11 bits
c = ((c1 & 0x1F) << 6) | (c2 & 0x3F)
if c < 0x80: # pragma: no cover
# Overlong encoding
skips.extend([i, i + 1]) # pragma: no cover
i += 2
continue
c3 = bytes[i + 2]
if (c1 & 0xF0 == 0xE0) and (c2 & 0xC0 == 0x80) and (c3 & 0xC0 == 0x80):
# U+0800 - U+FFFF - 16 bits
c = ((((c1 & 0x0F) << 6) | (c2 & 0x3F)) << 6) | (c3 & 0x3F)
if (c < 0x800) or (0xD800 <= c <= 0xDFFF):
# Overlong encoding or surrogate.
skips.extend([i, i + 1, i + 2])
i += 3
continue
c4 = bytes[i + 3]
if (
(c1 & 0xF8 == 0xF0)
and (c2 & 0xC0 == 0x80)
and (c3 & 0xC0 == 0x80)
and (c4 & 0xC0 == 0x80)
):
# U+10000 - U+10FFFF - 21 bits
c = ((((((c1 & 0x0F) << 6) | (c2 & 0x3F)) << 6) | (c3 & 0x3F)) << 6) | (
c4 & 0x3F
)
if (c < 0x10000) or (c > 0x10FFFF): # pragma: no cover
# Overlong encoding or invalid code point.
skips.extend([i, i + 1, i + 2, i + 3])
i += 4
continue
except IndexError:
pass
skips.append(i)
i += 1
return skips
# XXX backport: Another helper to support the Python 2 UTF-8 decoding hack.
def _chunks(b, indexes):
i = 0
for j in indexes:
yield b[i:j]
yield b[j : j + 1]
i = j + 1
yield b[i:]
def fs_encode(path):
"""
Encode a filesystem path to the proper filesystem encoding
@@ -257,7 +325,16 @@ def fs_encode(path):
if path is None:
raise TypeError("expected a valid path to encode")
if isinstance(path, six.text_type):
path = path.encode(_fs_encoding, _fs_encode_errors)
if six.PY2:
return b"".join(
(
_byte(ord(c) - 0xDC00)
if 0xDC00 <= ord(c) <= 0xDCFF
else c.encode(_fs_encoding, _fs_encode_errors)
)
for c in path
)
return path.encode(_fs_encoding, _fs_encode_errors)
return path
@@ -266,35 +343,49 @@ def fs_decode(path):
Decode a filesystem path using the proper filesystem encoding
:param path: The filesystem path to decode from bytes or string
:return: [description]
:rtype: [type]
:return: The filesystem path, decoded with the determined encoding
:rtype: Text
"""
path = _get_path(path)
if path is None:
raise TypeError("expected a valid path to decode")
if isinstance(path, six.binary_type):
path = path.decode(_fs_encoding, _fs_decode_errors)
if six.PY2:
from array import array
indexes = _invalid_utf8_indexes(array(str("B"), path))
return "".join(
chunk.decode(_fs_encoding, _fs_decode_errors)
for chunk in _chunks(path, indexes)
)
return path.decode(_fs_encoding, _fs_decode_errors)
return path
if sys.version_info >= (3, 3) and os.name != "nt":
_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
if sys.version_info[0] < 3:
_fs_encode_errors = "surrogateescape"
_fs_decode_errors = "surrogateescape"
_fs_encoding = "utf-8"
else:
_fs_encoding = "utf-8"
if six.PY3:
if os.name == "nt":
if sys.platform.startswith("win"):
_fs_error_fn = None
alt_strategy = "surrogatepass"
if sys.version_info[:2] > (3, 4):
alt_strategy = "surrogatepass"
else:
alt_strategy = "surrogateescape"
else:
if sys.version_info >= (3, 3):
_fs_encoding = sys.getfilesystemencoding()
if not _fs_encoding:
_fs_encoding = sys.getdefaultencoding()
alt_strategy = "surrogateescape"
_fs_error_fn = getattr(sys, "getfilesystemencodeerrors", None)
_fs_encode_errors = _fs_error_fn() if _fs_error_fn is not None else alt_strategy
_fs_decode_errors = _fs_error_fn() if _fs_error_fn is not None else alt_strategy
else:
_fs_encode_errors = "backslashreplace"
_fs_decode_errors = "replace"
_fs_encode_errors = _fs_error_fn() if _fs_error_fn else alt_strategy
_fs_decode_errors = _fs_error_fn() if _fs_error_fn else alt_strategy
_byte = chr if sys.version_info < (3,) else lambda i: bytes([i])
def to_native_string(string):
+43 -30
View File
@@ -222,13 +222,15 @@ def _create_subprocess(
c = _spawn_subprocess(
cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr
)
except Exception:
except Exception as exc:
import traceback
formatted_tb = "".join(traceback.format_exception(*sys.exc_info()))
sys.stderr.write("Error while executing command %s:" % " ".join(cmd._parts))
sys.stderr.write(formatted_tb)
raise
formatted_tb = "".join(traceback.format_exception(*sys.exc_info())) # pragma: no cover
sys.stderr.write( # pragma: no cover
"Error while executing command %s:" % to_native_string(" ".join(cmd._parts)) # pragma: no cover
) # pragma: no cover
sys.stderr.write(formatted_tb) # pragma: no cover
raise exc # pragma: no cover
if not block:
c.stdin.close()
spinner_orig_text = ""
@@ -397,14 +399,14 @@ def partialclass(cls, *args, **kwargs):
# Swiped from attrs.make_class
try:
type_.__module__ = sys._getframe(1).f_globals.get("__name__", "__main__")
except (AttributeError, ValueError):
pass
except (AttributeError, ValueError): # pragma: no cover
pass # pragma: no cover
return type_
# Borrowed from django -- force bytes and decode -- see link for details:
# https://github.com/django/django/blob/fc6b90b/django/utils/encoding.py#L112
def to_bytes(string, encoding="utf-8", errors="ignore"):
def to_bytes(string, encoding="utf-8", errors=None):
"""Force a value to bytes.
:param string: Some input that can be converted to a bytes.
@@ -415,16 +417,20 @@ def to_bytes(string, encoding="utf-8", errors="ignore"):
:rtype: bytes
"""
unicode_name = get_canonical_encoding_name("utf-8")
if not errors:
if encoding.lower() == "utf-8":
errors = "surrogateescape" if six.PY3 else "ignore"
if get_canonical_encoding_name(encoding) == unicode_name:
if six.PY3 and os.name == "nt":
errors = "surrogatepass"
else:
errors = "surrogateescape" if six.PY3 else "ignore"
else:
errors = "strict"
if isinstance(string, bytes):
if encoding.lower() == "utf-8":
if get_canonical_encoding_name(encoding) == unicode_name:
return string
else:
return string.decode("utf-8").encode(encoding, errors)
return string.decode(unicode_name).encode(encoding, errors)
elif isinstance(string, memoryview):
return bytes(string)
elif not isinstance(string, six.string_types):
@@ -452,9 +458,13 @@ def to_text(string, encoding="utf-8", errors=None):
:rtype: str
"""
unicode_name = get_canonical_encoding_name("utf-8")
if not errors:
if encoding.lower() == "utf-8":
errors = "surrogateescape" if six.PY3 else "ignore"
if get_canonical_encoding_name(encoding) == unicode_name:
if six.PY3 and os.name == "nt":
errors = "surrogatepass"
else:
errors = "surrogateescape" if six.PY3 else "ignore"
else:
errors = "strict"
if issubclass(type(string), six.text_type):
@@ -801,17 +811,16 @@ _color_stream_cache = WeakKeyDictionary()
if os.name == "nt" or sys.platform.startswith("win"):
def _wrap_for_color(stream, allow_color=True):
if colorama is not None:
if colorama is not None:
def _wrap_for_color(stream, color=None):
try:
cached = _color_stream_cache.get(stream)
except KeyError:
cached = None
if cached is not None:
return cached
if not _isatty(stream):
allow_color = False
_color_wrapper = colorama.AnsiToWin32(stream, strip=not allow_color)
strip = not _can_use_color(stream, color)
_color_wrapper = colorama.AnsiToWin32(stream, strip=strip)
result = _color_wrapper.stream
_write = result.write
@@ -829,8 +838,6 @@ if os.name == "nt" or sys.platform.startswith("win"):
pass
return result
return stream
def _cached_stream_lookup(stream_lookup_func, stream_resolution_func):
stream_cache = WeakKeyDictionary()
@@ -853,7 +860,7 @@ def _cached_stream_lookup(stream_lookup_func, stream_resolution_func):
return lookup
def get_text_stream(stream="stdout", encoding=None, allow_color=True):
def get_text_stream(stream="stdout", encoding=None):
"""Retrieve a unicode stream wrapper around **sys.stdout** or **sys.stderr**.
:param str stream: The name of the stream to wrap from the :mod:`sys` module.
@@ -916,15 +923,19 @@ def replace_with_text_stream(stream_name):
return None
def _can_use_color(stream=None, fg=None, bg=None, style=None):
if not any([fg, bg, style]):
def _can_use_color(stream=None, color=None):
from .termcolors import DISABLE_COLORS
if DISABLE_COLORS:
return False
if not color:
if not stream:
stream = sys.stdin
return _isatty(stream)
return any([fg, bg, style])
return bool(color)
def echo(text, fg=None, bg=None, style=None, file=None, err=False):
def echo(text, fg=None, bg=None, style=None, file=None, err=False, color=None):
"""Write the given text to the provided stream or **sys.stdout** by default.
Provides optional foreground and background colors from the ansi defaults:
@@ -939,6 +950,7 @@ def echo(text, fg=None, bg=None, style=None, file=None, err=False):
:param str bg: Foreground color to use (default: None)
:param str style: Style to use (default: None)
:param stream file: File to write to (default: None)
:param bool color: Whether to force color (i.e. ANSI codes are in the text)
"""
if file and not hasattr(file, "write"):
@@ -963,12 +975,13 @@ def echo(text, fg=None, bg=None, style=None, file=None, err=False):
buffer.flush()
return
if text and not is_bytes(text):
can_use_color = _can_use_color(file, fg=fg, bg=bg, style=style)
if os.name == "nt":
can_use_color = _can_use_color(file, color=color)
if any([fg, bg, style]):
text = colorize(text, fg=fg, bg=bg, attrs=style)
file = _wrap_for_color(file, allow_color=can_use_color)
elif not can_use_color:
if not can_use_color or (os.name == "nt" and not _wrap_for_color):
text = ANSI_REMOVAL_RE.sub("", text)
elif os.name == "nt" and _wrap_for_color:
file = _wrap_for_color(file, color=color)
if text:
file.write(text)
file.flush()
+27
View File
@@ -306,6 +306,22 @@ def create_tracked_tempfile(*args, **kwargs):
return _NamedTemporaryFile(*args, **kwargs)
def _find_icacls_exe():
if os.name == "nt":
paths = [
os.path.expandvars(r"%windir%\{0}").format(subdir)
for subdir in ("system32", "SysWOW64")
]
for path in paths:
icacls_path = next(
iter(fn for fn in os.listdir(path) if fn.lower() == "icacls.exe"), None
)
if icacls_path is not None:
icacls_path = os.path.join(path, icacls_path)
return icacls_path
return None
def set_write_bit(fn):
# type: (str) -> None
"""
@@ -321,6 +337,17 @@ def set_write_bit(fn):
return
file_stat = os.stat(fn).st_mode
os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
if os.name == "nt":
from ._winconsole import get_current_user
user_sid = get_current_user()
icacls_exe = _find_icacls_exe() or "icacls"
from .misc import run
if user_sid:
_, err = run([icacls_exe, "/grant", "{0}:WD".format(user_sid), "''{0}''".format(fn), "/T", "/C", "/Q"])
if not err:
return
if not os.path.isdir(fn):
for path in [fn, os.path.dirname(fn)]:
try:
+1 -1
View File
@@ -129,7 +129,7 @@ def build_dists(ctx):
log('Building sdist using %s ....' % executable)
os.environ["PIPENV_PYTHON"] = py_version
ctx.run('pipenv install --dev', env=env)
ctx.run('pipenv run pip install -e . --upgrade --upgrade-strategy=eager', env=env)
ctx.run('pipenv run pip install -e . --upgrade --upgrade-strategy=eager --no-use-pep517', env=env)
log('Building wheel using python %s ....' % py_version)
if py_version == '3.6':
ctx.run('pipenv run python setup.py sdist bdist_wheel', env=env)