From 39f384891f7208876df6f755bd7a28b1ebc00a1c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 11 Jun 2019 12:19:11 -0400 Subject: [PATCH] Fix marker parsing for broken marker format - Fixes #3786 Signed-off-by: Dan Ryan --- news/3786.bugfix.rst | 1 + pipenv/utils.py | 54 +++--- .../vendor/requirementslib/models/markers.py | 171 ++++++++++++------ .../requirementslib/models/setup_info.py | 18 +- pipenv/vendor/requirementslib/models/utils.py | 21 ++- pipenv/vendor/vistir/misc.py | 1 - 6 files changed, 184 insertions(+), 82 deletions(-) create mode 100644 news/3786.bugfix.rst diff --git a/news/3786.bugfix.rst b/news/3786.bugfix.rst new file mode 100644 index 00000000..210f7973 --- /dev/null +++ b/news/3786.bugfix.rst @@ -0,0 +1 @@ +Resolved an issue which caused resolution to fail when encountering poorly formatted ``python_version`` markers in ``setup.py`` and ``setup.cfg`` files. diff --git a/pipenv/utils.py b/pipenv/utils.py index fc9df248..060b8532 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -703,14 +703,16 @@ class Resolver(object): return None def resolve_constraints(self): + from .vendor.requirementslib.models.markers import marker_from_specifier new_tree = set() for result in self.resolved_tree: if result.markers: self.markers[result.name] = result.markers else: candidate = self.fetch_candidate(result) - if getattr(candidate, "requires_python", None): - marker = make_marker_from_specifier(candidate.requires_python) + requires_python = getattr(candidate, "requires_python", None) + if requires_python: + marker = marker_from_specifier(candidate.requires_python) self.markers[result.name] = marker result.markers = marker if result.req: @@ -812,7 +814,7 @@ class Resolver(object): # This fixes a race condition in resolution for missing dependency caches # see pypa/pipenv#3289 if not self._should_include_hash(ireq): - return set() + return add_to_set(set(), ireq_hashes) elif self._should_include_hash(ireq) and ( not ireq_hashes or ireq.link.scheme == "file" ): @@ -2083,27 +2085,33 @@ def is_python_command(line): return False -def make_marker_from_specifier(spec): - # type: (str) -> Optional[Marker] - """Given a python version specifier, create a marker +# def make_marker_from_specifier(spec): +# # type: (str) -> Optional[Marker] +# """Given a python version specifier, create a marker - :param spec: A specifier - :type spec: str - :return: A new marker - :rtype: Optional[:class:`packaging.marker.Marker`] - """ - from .vendor.packaging.specifiers import SpecifierSet, Specifier - from .vendor.packaging.markers import Marker - from .vendor.requirementslib.models.markers import cleanup_pyspecs, format_pyversion - if not any(spec.startswith(k) for k in Specifier._operators.keys()): - if spec.strip().lower() in ["any", "", "*"]: - return None - spec = "=={0}".format(spec) - elif spec.startswith("==") and spec.count("=") > 3: - spec = "=={0}".format(spec.lstrip("=")) - specset = cleanup_pyspecs(SpecifierSet(spec)) - marker_str = " and ".join([format_pyversion(pv) for pv in specset]) - return Marker(marker_str) +# :param spec: A specifier +# :type spec: str +# :return: A new marker +# :rtype: Optional[:class:`packaging.marker.Marker`] +# """ +# from .vendor.packaging.markers import Marker +# from .vendor.packaging.specifiers import SpecifierSet, Specifier +# from .vendor.requirementslib.models.markers import cleanup_pyspecs, format_pyversion +# if not any(spec.startswith(k) for k in Specifier._operators.keys()): +# if spec.strip().lower() in ["any", "", "*"]: +# return None +# spec = "=={0}".format(spec) +# elif spec.startswith("==") and spec.count("=") > 3: +# spec = "=={0}".format(spec.lstrip("=")) +# if not spec: +# return None +# marker_segments = [] +# print(spec) +# for marker_segment in cleanup_pyspecs(spec): +# print(marker_segment) +# marker_segments.append(format_pyversion(marker_segment)) +# marker_str = " and ".join(marker_segments) +# return Marker(marker_str) @contextlib.contextmanager diff --git a/pipenv/vendor/requirementslib/models/markers.py b/pipenv/vendor/requirementslib/models/markers.py index e1014917..5e665114 100644 --- a/pipenv/vendor/requirementslib/models/markers.py +++ b/pipenv/vendor/requirementslib/models/markers.py @@ -7,7 +7,7 @@ import distlib.markers import packaging.version import six from packaging.markers import InvalidMarker, Marker -from packaging.specifiers import Specifier, SpecifierSet +from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet from vistir.compat import Mapping, Set, lru_cache from vistir.misc import dedup @@ -19,7 +19,20 @@ from six.moves import reduce # isort:skip if MYPY_RUNNING: - from typing import Optional, List, Type, Any + from typing import ( + Optional, + List, + Type, + Any, + Tuple, + Union, + Set, + AnyStr, + Text, + Iterator, + ) + + STRING_TYPE = Union[str, bytes, Text] MAX_VERSIONS = {2: 7, 3: 10} @@ -117,13 +130,15 @@ class PipenvMarkers(object): return combined_marker -@lru_cache(maxsize=128) +@lru_cache(maxsize=1024) def _tuplize_version(version): + # type: (STRING_TYPE) -> Tuple[int, ...] return tuple(int(x) for x in filter(lambda i: i != "*", version.split("."))) -@lru_cache(maxsize=128) +@lru_cache(maxsize=1024) def _format_version(version): + # type: (Tuple[int, ...]) -> STRING_TYPE if not isinstance(version, six.string_types): return ".".join(str(i) for i in version) return version @@ -133,8 +148,9 @@ def _format_version(version): REPLACE_RANGES = {">": ">=", "<=": "<"} -@lru_cache(maxsize=128) +@lru_cache(maxsize=1024) def _format_pyspec(specifier): + # type: (Union[STRING_TYPE, Specifier]) -> Specifier if isinstance(specifier, str): if not any(op in specifier for op in Specifier._operators.keys()): specifier = "=={0}".format(specifier) @@ -165,7 +181,7 @@ def _format_pyspec(specifier): return specifier -@lru_cache(maxsize=128) +@lru_cache(maxsize=1024) def _get_specs(specset): if specset is None: return @@ -191,8 +207,8 @@ def _get_specs(specset): return sorted(result, key=operator.itemgetter(1)) -@lru_cache(maxsize=128) def _group_by_op(specs): + # type: (Union[Set[Specifier], SpecifierSet]) -> Iterator specs = [_get_specs(x) for x in list(specs)] flattened = [(op, version) for spec in specs for op, version in spec] specs = sorted(flattened) @@ -200,62 +216,92 @@ def _group_by_op(specs): return grouping -@lru_cache(maxsize=128) +def normalize_specifier_set(specs): + # type: (Union[str, SpecifierSet]) -> Optional[Set[Specifier]] + """Given a specifier set, a string, or an iterable, normalize the specifiers + + .. note:: This function exists largely to deal with ``pyzmq`` which handles + the ``requires_python`` specifier incorrectly, using ``3.7*`` rather than + the correct form of ``3.7.*``. This workaround can likely go away if + we ever introduce enforcement for metadata standards on PyPI. + + :param Union[str, SpecifierSet] specs: Supplied specifiers to normalize + :return: A new set of specifiers or specifierset + :rtype: Union[Set[Specifier], :class:`~packaging.specifiers.SpecifierSet`] + """ + if not specs: + return None + if isinstance(specs, set): + return specs + # when we aren't dealing with a string at all, we can normalize this as usual + elif not isinstance(specs, six.string_types): + return {_format_pyspec(spec) for spec in specs} + spec_list = [] + for spec in specs.split(","): + if spec.endswith(".*"): + spec = spec.rstrip(".*") + elif spec.endswith("*"): + spec = spec.rstrip("*") + spec_list.append(spec) + return normalize_specifier_set(SpecifierSet(",".join(spec_list))) + + +def get_sorted_version_string(version_set): + # type: (Set[AnyStr]) -> AnyStr + version_list = sorted( + "{0}".format(_format_version(version)) for version in version_set + ) + version = ", ".join(version_list) + return version + + +@lru_cache(maxsize=1024) def cleanup_pyspecs(specs, joiner="or"): - if isinstance(specs, six.string_types): - specs = set([_format_pyspec(specs)]) - else: - specs = {_format_pyspec(spec) for spec in specs} + specs = normalize_specifier_set(specs) # for != operator we want to group by version # if all are consecutive, join as a list - results = set() - for op, versions in _group_by_op(tuple(specs)): - versions = [version[1] for version in versions] - versions = sorted(dedup(versions)) + results = {} + translation_map = { # if we are doing an or operation, we need to use the min for >= # this way OR(>=2.6, >=2.7, >=3.6) picks >=2.6 # if we do an AND operation we need to use MAX to be more selective - if op in (">", ">="): - if joiner == "or": - results.add((op, _format_version(min(versions)))) - else: - results.add((op, _format_version(max(versions)))) + (">", ">="): { + "or": lambda x: _format_version(min(x)), + "and": lambda x: _format_version(max(x)), + }, # we use inverse logic here so we will take the max value if we are # using OR but the min value if we are using AND - elif op in ("<=", "<"): - if joiner == "or": - results.add((op, _format_version(max(versions)))) - else: - results.add((op, _format_version(min(versions)))) + ("<", "<="): { + "or": lambda x: _format_version(max(x)), + "and": lambda x: _format_version(min(x)), + }, # leave these the same no matter what operator we use - elif op in ("!=", "==", "~="): - version_list = sorted( - "{0}".format(_format_version(version)) for version in versions - ) - version = ", ".join(version_list) - if len(version_list) == 1: - results.add((op, version)) - elif op == "!=": - results.add(("not in", version)) - elif op == "==": - results.add(("in", version)) - else: - specifier = SpecifierSet( - ",".join(sorted("{0}{1}".format(op, v) for v in version_list)) - )._specs - for s in specifier: - results.add((s._spec[0], s._spec[1])) - else: - if len(version) == 1: - results.add((op, version)) - else: - specifier = SpecifierSet("{0}".format(version))._specs - for s in specifier: - results.add((s._spec[0], s._spec[1])) - return sorted(results, key=operator.itemgetter(1)) + ("!=", "==", "~=", "==="): { + "or": lambda x: get_sorted_version_string(x), + "and": lambda x: get_sorted_version_string(x), + }, + } + op_translations = { + "!=": lambda x: "not in" if len(x) > 1 else "!=", + "==": lambda x: "in" if len(x) > 1 else "==", + } + translation_keys = list(translation_map.keys()) + for op, versions in _group_by_op(tuple(specs)): + versions = [version[1] for version in versions] + versions = sorted(dedup(versions)) + op_key = next(iter(k for k in translation_keys if op in k), None) + version_value = versions + if op_key is not None: + version_value = translation_map[op_key][joiner](versions) + if op in op_translations: + op = op_translations[op](versions) + results[op] = version_value + return sorted([(k, v) for k, v in results.items()], key=operator.itemgetter(1)) +@lru_cache(maxsize=1024) def fix_version_tuple(version_tuple): + # type: (Tuple[AnyStr, AnyStr]) -> Tuple[AnyStr, AnyStr] op, version = version_tuple max_major = max(MAX_VERSIONS.keys()) if version[0] > max_major: @@ -269,6 +315,7 @@ def fix_version_tuple(version_tuple): @lru_cache(maxsize=128) def get_versions(specset, group_by_operator=True): + # type: (Union[Set[Specifier], SpecifierSet], bool) -> List[Tuple[STRING_TYPE, STRING_TYPE]] specs = [_get_specs(x) for x in list(tuple(specset))] initial_sort_key = lambda k: (k[0], k[1]) initial_grouping_key = operator.itemgetter(0) @@ -292,12 +339,14 @@ def get_versions(specset, group_by_operator=True): def _ensure_marker(marker): + # type: (Union[STRING_TYPE, Marker]) -> Marker if not is_instance(marker, Marker): return Marker(str(marker)) return marker def gen_marker(mkr): + # type: (List[STRING_TYPE]) -> Marker m = Marker("python_version == '1'") m._markers.pop() m._markers.append(mkr) @@ -437,6 +486,7 @@ def get_contained_extras(marker): return extras +@lru_cache(maxsize=1024) def get_contained_pyversions(marker): """Collect all `python_version` operands from a marker. """ @@ -596,6 +646,7 @@ def format_pyversion(parts): def normalize_marker_str(marker): + # type: (Union[Marker, STRING_TYPE]) marker_str = "" if not marker: return None @@ -612,3 +663,21 @@ def normalize_marker_str(marker): else: marker_str = "{0!s}".format(marker) return marker_str.replace('"', "'") + + +@lru_cache(maxsize=1024) +def marker_from_specifier(spec): + # type: (STRING_TYPE) -> Marker + if not any(spec.startswith(k) for k in Specifier._operators.keys()): + if spec.strip().lower() in ["any", "", "*"]: + return None + spec = "=={0}".format(spec) + elif spec.startswith("==") and spec.count("=") > 3: + spec = "=={0}".format(spec.lstrip("=")) + if not spec: + return None + marker_segments = [] + for marker_segment in cleanup_pyspecs(spec): + marker_segments.append(format_pyversion(marker_segment)) + marker_str = " and ".join(marker_segments).replace('"', "'") + return Marker(marker_str) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 0dd5b3c8..454cbfcc 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -33,6 +33,7 @@ from .utils import ( get_name_variants, get_pyproject, init_requirement, + read_source, split_vcs_method_from_uri, strip_extras_markers_from_requirement, ) @@ -136,10 +137,15 @@ class BuildEnv(pep517.envbuild.BuildEnvironment): class HookCaller(pep517.wrappers.Pep517HookCaller): - def __init__(self, source_dir, build_backend): + def __init__(self, source_dir, build_backend, backend_path=None): self.source_dir = os.path.abspath(source_dir) self.build_backend = build_backend self._subprocess_runner = pep517_subprocess_runner + if backend_path: + backend_path = [ + pep517.wrappers.norm_and_check(self.source_dir, p) for p in backend_path + ] + self.backend_path = backend_path def parse_special_directives(setup_entry, package_dir=None): @@ -151,8 +157,7 @@ def parse_special_directives(setup_entry, package_dir=None): _, path = setup_entry.split("file:") path = path.strip() if os.path.exists(path): - with open(path, "r") as fh: - rv = fh.read() + rv = read_source(path) elif setup_entry.startswith("attr:"): _, resource = setup_entry.split("attr:") resource = resource.strip() @@ -660,7 +665,9 @@ class Analyzer(ast.NodeVisitor): def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # noqa:C901 # type: (Any, bool, Optional[Analyzer], bool) -> Union[List[Any], Dict[Any, Any], Tuple[Any, ...], STRING_TYPE] - unparse = partial(ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer, recurse=recurse) + unparse = partial( + ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer, recurse=recurse + ) if isinstance(item, ast.Dict): unparsed = dict(zip(unparse(item.keys), unparse(item.values))) elif isinstance(item, ast.List): @@ -770,8 +777,7 @@ def ast_parse_attribute_from_file(path, attribute): def ast_parse_file(path): # type: (S) -> Analyzer - with open(path, "r") as fh: - tree = ast.parse(fh.read()) + tree = ast.parse(read_source(path)) ast_analyzer = Analyzer() ast_analyzer.visit(tree) return ast_analyzer diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 2f4c26c2..fd5567a8 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -541,7 +541,7 @@ def split_ref_from_uri(uri): parsed = urllib_parse.urlparse(uri) path = parsed.path ref = None - if "@" in path: + if parsed.scheme != "file" and "@" in path: path, _, ref = path.rpartition("@") parsed = parsed._replace(path=path) return (urllib_parse.urlunparse(parsed), ref) @@ -968,6 +968,25 @@ def get_name_variants(pkg): return names +def read_source(path, encoding="utf-8"): + # type: (S, S) -> S + """ + Read a source file and get the contents with proper encoding for Python 2/3. + + :param AnyStr path: the file path + :param AnyStr encoding: the encoding that defaults to UTF-8 + :returns: The contents of the source file + :rtype: AnyStr + """ + if six.PY3: + with open(path, "r", encoding=encoding) as fp: + return fp.read() + else: + with open(path, "r") as fp: + return fp.read() + + + SETUPTOOLS_SHIM = ( "import setuptools, tokenize;__file__=%r;" "f=getattr(tokenize, 'open', open)(__file__);" diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 8d3322d2..36218a50 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -6,7 +6,6 @@ import json import locale import logging import os -import signal import subprocess import sys from collections import OrderedDict