Fix marker parsing for broken marker format

- Fixes #3786 Signed-off-by: Dan Ryan <dan@danryan.co>
2026-06-05 22:50:18 +00:00 · 2019-06-11 12:19:11 -04:00
parent c0f1f7523f
commit 39f384891f
6 changed files with 184 additions and 82 deletions
@@ -0,0 +1 @@
+Resolved an issue which caused resolution to fail when encountering poorly formatted ``python_version`` markers in ``setup.py`` and ``setup.cfg`` files.
@@ -703,14 +703,16 @@ class Resolver(object):
        return None

    def resolve_constraints(self):
+        from .vendor.requirementslib.models.markers import marker_from_specifier
        new_tree = set()
        for result in self.resolved_tree:
            if result.markers:
                self.markers[result.name] = result.markers
            else:
                candidate = self.fetch_candidate(result)
-                if getattr(candidate, "requires_python", None):
-                    marker = make_marker_from_specifier(candidate.requires_python)
+                requires_python = getattr(candidate, "requires_python", None)
+                if requires_python:
+                    marker = marker_from_specifier(candidate.requires_python)
                    self.markers[result.name] = marker
                    result.markers = marker
                    if result.req:
@@ -812,7 +814,7 @@ class Resolver(object):
        # This fixes a race condition in resolution for missing dependency caches
        # see pypa/pipenv#3289
        if not self._should_include_hash(ireq):
-            return set()
+            return add_to_set(set(), ireq_hashes)
        elif self._should_include_hash(ireq) and (
            not ireq_hashes or ireq.link.scheme == "file"
        ):
@@ -2083,27 +2085,33 @@ def is_python_command(line):
    return False


-def make_marker_from_specifier(spec):
-    # type: (str) -> Optional[Marker]
-    """Given a python version specifier, create a marker
+# def make_marker_from_specifier(spec):
+#     # type: (str) -> Optional[Marker]
+#     """Given a python version specifier, create a marker

-    :param spec: A specifier
-    :type spec: str
-    :return: A new marker
-    :rtype: Optional[:class:`packaging.marker.Marker`]
-    """
-    from .vendor.packaging.specifiers import SpecifierSet, Specifier
-    from .vendor.packaging.markers import Marker
-    from .vendor.requirementslib.models.markers import cleanup_pyspecs, format_pyversion
-    if not any(spec.startswith(k) for k in Specifier._operators.keys()):
-        if spec.strip().lower() in ["any", "<any>", "*"]:
-            return None
-        spec = "=={0}".format(spec)
-    elif spec.startswith("==") and spec.count("=") > 3:
-        spec = "=={0}".format(spec.lstrip("="))
-    specset = cleanup_pyspecs(SpecifierSet(spec))
-    marker_str = " and ".join([format_pyversion(pv) for pv in specset])
-    return Marker(marker_str)
+#     :param spec: A specifier
+#     :type spec: str
+#     :return: A new marker
+#     :rtype: Optional[:class:`packaging.marker.Marker`]
+#     """
+#     from .vendor.packaging.markers import Marker
+#     from .vendor.packaging.specifiers import SpecifierSet, Specifier
+#     from .vendor.requirementslib.models.markers import cleanup_pyspecs, format_pyversion
+#     if not any(spec.startswith(k) for k in Specifier._operators.keys()):
+#         if spec.strip().lower() in ["any", "<any>", "*"]:
+#             return None
+#         spec = "=={0}".format(spec)
+#     elif spec.startswith("==") and spec.count("=") > 3:
+#         spec = "=={0}".format(spec.lstrip("="))
+#     if not spec:
+#         return None
+#     marker_segments = []
+#     print(spec)
+#     for marker_segment in cleanup_pyspecs(spec):
+#         print(marker_segment)
+#         marker_segments.append(format_pyversion(marker_segment))
+#     marker_str = " and ".join(marker_segments)
+#     return Marker(marker_str)


@contextlib.contextmanager
@@ -7,7 +7,7 @@ import distlib.markers
 import packaging.version
 import six
 from packaging.markers import InvalidMarker, Marker
-from packaging.specifiers import Specifier, SpecifierSet
+from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet
 from vistir.compat import Mapping, Set, lru_cache
 from vistir.misc import dedup

@@ -19,7 +19,20 @@ from six.moves import reduce  # isort:skip


 if MYPY_RUNNING:
-    from typing import Optional, List, Type, Any
+    from typing import (
+        Optional,
+        List,
+        Type,
+        Any,
+        Tuple,
+        Union,
+        Set,
+        AnyStr,
+        Text,
+        Iterator,
+    )
+
+    STRING_TYPE = Union[str, bytes, Text]


 MAX_VERSIONS = {2: 7, 3: 10}
@@ -117,13 +130,15 @@ class PipenvMarkers(object):
            return combined_marker


-@lru_cache(maxsize=128)
+@lru_cache(maxsize=1024)
 def _tuplize_version(version):
+    # type: (STRING_TYPE) -> Tuple[int, ...]
    return tuple(int(x) for x in filter(lambda i: i != "*", version.split(".")))


-@lru_cache(maxsize=128)
+@lru_cache(maxsize=1024)
 def _format_version(version):
+    # type: (Tuple[int, ...]) -> STRING_TYPE
    if not isinstance(version, six.string_types):
        return ".".join(str(i) for i in version)
    return version
@@ -133,8 +148,9 @@ def _format_version(version):
 REPLACE_RANGES = {">": ">=", "<=": "<"}


-@lru_cache(maxsize=128)
+@lru_cache(maxsize=1024)
 def _format_pyspec(specifier):
+    # type: (Union[STRING_TYPE, Specifier]) -> Specifier
    if isinstance(specifier, str):
        if not any(op in specifier for op in Specifier._operators.keys()):
            specifier = "=={0}".format(specifier)
@@ -165,7 +181,7 @@ def _format_pyspec(specifier):
    return specifier


-@lru_cache(maxsize=128)
+@lru_cache(maxsize=1024)
 def _get_specs(specset):
    if specset is None:
        return
@@ -191,8 +207,8 @@ def _get_specs(specset):
    return sorted(result, key=operator.itemgetter(1))


-@lru_cache(maxsize=128)
 def _group_by_op(specs):
+    # type: (Union[Set[Specifier], SpecifierSet]) -> Iterator
    specs = [_get_specs(x) for x in list(specs)]
    flattened = [(op, version) for spec in specs for op, version in spec]
    specs = sorted(flattened)
@@ -200,62 +216,92 @@ def _group_by_op(specs):
    return grouping


-@lru_cache(maxsize=128)
+def normalize_specifier_set(specs):
+    # type: (Union[str, SpecifierSet]) -> Optional[Set[Specifier]]
+    """Given a specifier set, a string, or an iterable, normalize the specifiers
+
+    .. note:: This function exists largely to deal with ``pyzmq`` which handles
+        the ``requires_python`` specifier incorrectly, using ``3.7*`` rather than
+        the correct form of ``3.7.*``.  This workaround can likely go away if
+        we ever introduce enforcement for metadata standards on PyPI.
+
+    :param Union[str, SpecifierSet] specs: Supplied specifiers to normalize
+    :return: A new set of specifiers or specifierset
+    :rtype: Union[Set[Specifier], :class:`~packaging.specifiers.SpecifierSet`]
+    """
+    if not specs:
+        return None
+    if isinstance(specs, set):
+        return specs
+    # when we aren't dealing with a string at all, we can normalize this as usual
+    elif not isinstance(specs, six.string_types):
+        return {_format_pyspec(spec) for spec in specs}
+    spec_list = []
+    for spec in specs.split(","):
+        if spec.endswith(".*"):
+            spec = spec.rstrip(".*")
+        elif spec.endswith("*"):
+            spec = spec.rstrip("*")
+        spec_list.append(spec)
+    return normalize_specifier_set(SpecifierSet(",".join(spec_list)))
+
+
+def get_sorted_version_string(version_set):
+    # type: (Set[AnyStr]) -> AnyStr
+    version_list = sorted(
+        "{0}".format(_format_version(version)) for version in version_set
+    )
+    version = ", ".join(version_list)
+    return version
+
+
+@lru_cache(maxsize=1024)
 def cleanup_pyspecs(specs, joiner="or"):
-    if isinstance(specs, six.string_types):
-        specs = set([_format_pyspec(specs)])
-    else:
-        specs = {_format_pyspec(spec) for spec in specs}
+    specs = normalize_specifier_set(specs)
    # for != operator we want to group by version
    # if all are consecutive, join as a list
-    results = set()
-    for op, versions in _group_by_op(tuple(specs)):
-        versions = [version[1] for version in versions]
-        versions = sorted(dedup(versions))
+    results = {}
+    translation_map = {
        # if we are doing an or operation, we need to use the min for >=
        # this way OR(>=2.6, >=2.7, >=3.6) picks >=2.6
        # if we do an AND operation we need to use MAX to be more selective
-        if op in (">", ">="):
-            if joiner == "or":
-                results.add((op, _format_version(min(versions))))
-            else:
-                results.add((op, _format_version(max(versions))))
+        (">", ">="): {
+            "or": lambda x: _format_version(min(x)),
+            "and": lambda x: _format_version(max(x)),
+        },
        # we use inverse logic here so we will take the max value if we are
        # using OR but the min value if we are using AND
-        elif op in ("<=", "<"):
-            if joiner == "or":
-                results.add((op, _format_version(max(versions))))
-            else:
-                results.add((op, _format_version(min(versions))))
+        ("<", "<="): {
+            "or": lambda x: _format_version(max(x)),
+            "and": lambda x: _format_version(min(x)),
+        },
        # leave these the same no matter what operator we use
-        elif op in ("!=", "==", "~="):
-            version_list = sorted(
-                "{0}".format(_format_version(version)) for version in versions
-            )
-            version = ", ".join(version_list)
-            if len(version_list) == 1:
-                results.add((op, version))
-            elif op == "!=":
-                results.add(("not in", version))
-            elif op == "==":
-                results.add(("in", version))
-            else:
-                specifier = SpecifierSet(
-                    ",".join(sorted("{0}{1}".format(op, v) for v in version_list))
-                )._specs
-                for s in specifier:
-                    results.add((s._spec[0], s._spec[1]))
-        else:
-            if len(version) == 1:
-                results.add((op, version))
-            else:
-                specifier = SpecifierSet("{0}".format(version))._specs
-                for s in specifier:
-                    results.add((s._spec[0], s._spec[1]))
-    return sorted(results, key=operator.itemgetter(1))
+        ("!=", "==", "~=", "==="): {
+            "or": lambda x: get_sorted_version_string(x),
+            "and": lambda x: get_sorted_version_string(x),
+        },
+    }
+    op_translations = {
+        "!=": lambda x: "not in" if len(x) > 1 else "!=",
+        "==": lambda x: "in" if len(x) > 1 else "==",
+    }
+    translation_keys = list(translation_map.keys())
+    for op, versions in _group_by_op(tuple(specs)):
+        versions = [version[1] for version in versions]
+        versions = sorted(dedup(versions))
+        op_key = next(iter(k for k in translation_keys if op in k), None)
+        version_value = versions
+        if op_key is not None:
+            version_value = translation_map[op_key][joiner](versions)
+        if op in op_translations:
+            op = op_translations[op](versions)
+        results[op] = version_value
+    return sorted([(k, v) for k, v in results.items()], key=operator.itemgetter(1))


+@lru_cache(maxsize=1024)
 def fix_version_tuple(version_tuple):
+    # type: (Tuple[AnyStr, AnyStr]) -> Tuple[AnyStr, AnyStr]
    op, version = version_tuple
    max_major = max(MAX_VERSIONS.keys())
    if version[0] > max_major:
@@ -269,6 +315,7 @@ def fix_version_tuple(version_tuple):

@lru_cache(maxsize=128)
 def get_versions(specset, group_by_operator=True):
+    # type: (Union[Set[Specifier], SpecifierSet], bool) -> List[Tuple[STRING_TYPE, STRING_TYPE]]
    specs = [_get_specs(x) for x in list(tuple(specset))]
    initial_sort_key = lambda k: (k[0], k[1])
    initial_grouping_key = operator.itemgetter(0)
@@ -292,12 +339,14 @@ def get_versions(specset, group_by_operator=True):


 def _ensure_marker(marker):
+    # type: (Union[STRING_TYPE, Marker]) -> Marker
    if not is_instance(marker, Marker):
        return Marker(str(marker))
    return marker


 def gen_marker(mkr):
+    # type: (List[STRING_TYPE]) -> Marker
    m = Marker("python_version == '1'")
    m._markers.pop()
    m._markers.append(mkr)
@@ -437,6 +486,7 @@ def get_contained_extras(marker):
    return extras


+@lru_cache(maxsize=1024)
 def get_contained_pyversions(marker):
    """Collect all `python_version` operands from a marker.
    """
@@ -596,6 +646,7 @@ def format_pyversion(parts):


 def normalize_marker_str(marker):
+    # type: (Union[Marker, STRING_TYPE])
    marker_str = ""
    if not marker:
        return None
@@ -612,3 +663,21 @@ def normalize_marker_str(marker):
        else:
            marker_str = "{0!s}".format(marker)
    return marker_str.replace('"', "'")
+
+
+@lru_cache(maxsize=1024)
+def marker_from_specifier(spec):
+    # type: (STRING_TYPE) -> Marker
+    if not any(spec.startswith(k) for k in Specifier._operators.keys()):
+        if spec.strip().lower() in ["any", "<any>", "*"]:
+            return None
+        spec = "=={0}".format(spec)
+    elif spec.startswith("==") and spec.count("=") > 3:
+        spec = "=={0}".format(spec.lstrip("="))
+    if not spec:
+        return None
+    marker_segments = []
+    for marker_segment in cleanup_pyspecs(spec):
+        marker_segments.append(format_pyversion(marker_segment))
+    marker_str = " and ".join(marker_segments).replace('"', "'")
+    return Marker(marker_str)
@@ -33,6 +33,7 @@ from .utils import (
    get_name_variants,
    get_pyproject,
    init_requirement,
+    read_source,
    split_vcs_method_from_uri,
    strip_extras_markers_from_requirement,
 )
@@ -136,10 +137,15 @@ class BuildEnv(pep517.envbuild.BuildEnvironment):


 class HookCaller(pep517.wrappers.Pep517HookCaller):
-    def __init__(self, source_dir, build_backend):
+    def __init__(self, source_dir, build_backend, backend_path=None):
        self.source_dir = os.path.abspath(source_dir)
        self.build_backend = build_backend
        self._subprocess_runner = pep517_subprocess_runner
+        if backend_path:
+            backend_path = [
+                pep517.wrappers.norm_and_check(self.source_dir, p) for p in backend_path
+            ]
+        self.backend_path = backend_path


 def parse_special_directives(setup_entry, package_dir=None):
@@ -151,8 +157,7 @@ def parse_special_directives(setup_entry, package_dir=None):
        _, path = setup_entry.split("file:")
        path = path.strip()
        if os.path.exists(path):
-            with open(path, "r") as fh:
-                rv = fh.read()
+            rv = read_source(path)
    elif setup_entry.startswith("attr:"):
        _, resource = setup_entry.split("attr:")
        resource = resource.strip()
@@ -660,7 +665,9 @@ class Analyzer(ast.NodeVisitor):

 def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True):  # noqa:C901
    # type: (Any, bool, Optional[Analyzer], bool) -> Union[List[Any], Dict[Any, Any], Tuple[Any, ...], STRING_TYPE]
-    unparse = partial(ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer, recurse=recurse)
+    unparse = partial(
+        ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer, recurse=recurse
+    )
    if isinstance(item, ast.Dict):
        unparsed = dict(zip(unparse(item.keys), unparse(item.values)))
    elif isinstance(item, ast.List):
@@ -770,8 +777,7 @@ def ast_parse_attribute_from_file(path, attribute):

 def ast_parse_file(path):
    # type: (S) -> Analyzer
-    with open(path, "r") as fh:
-        tree = ast.parse(fh.read())
+    tree = ast.parse(read_source(path))
    ast_analyzer = Analyzer()
    ast_analyzer.visit(tree)
    return ast_analyzer
@@ -541,7 +541,7 @@ def split_ref_from_uri(uri):
    parsed = urllib_parse.urlparse(uri)
    path = parsed.path
    ref = None
-    if "@" in path:
+    if parsed.scheme != "file" and "@" in path:
        path, _, ref = path.rpartition("@")
    parsed = parsed._replace(path=path)
    return (urllib_parse.urlunparse(parsed), ref)
@@ -968,6 +968,25 @@ def get_name_variants(pkg):
    return names


+def read_source(path, encoding="utf-8"):
+    # type: (S, S) -> S
+    """
+    Read a source file and get the contents with proper encoding for Python 2/3.
+
+    :param AnyStr path: the file path
+    :param AnyStr encoding: the encoding that defaults to UTF-8
+    :returns: The contents of the source file
+    :rtype: AnyStr
+    """
+    if six.PY3:
+        with open(path, "r", encoding=encoding) as fp:
+            return fp.read()
+    else:
+        with open(path, "r") as fp:
+            return fp.read()
+
+
+
 SETUPTOOLS_SHIM = (
    "import setuptools, tokenize;__file__=%r;"
    "f=getattr(tokenize, 'open', open)(__file__);"
@@ -6,7 +6,6 @@ import json
 import locale
 import logging
 import os
-import signal
 import subprocess
 import sys
 from collections import OrderedDict
				`@@ -0,0 +1 @@`
				Resolved an issue which caused resolution to fail when encountering poorly formatted ``python_version`` markers in ``setup.py`` and ``setup.cfg`` files.