From d2fe28a8b0a0f64dbc90c6e5fc27d28e0f2df967 Mon Sep 17 00:00:00 2001 From: Oz N Tiram Date: Sun, 2 Oct 2022 23:33:16 +0200 Subject: [PATCH] Bump dparse version to 0.6.2 --- pipenv/vendor/dparse/__init__.py | 2 +- pipenv/vendor/dparse/dependencies.py | 50 +++++- pipenv/vendor/dparse/errors.py | 13 +- pipenv/vendor/dparse/filetypes.py | 1 + pipenv/vendor/dparse/parser.py | 168 +++++++++++++----- pipenv/vendor/dparse/regex.py | 37 +--- pipenv/vendor/dparse/updater.py | 27 +-- pipenv/vendor/vendor.txt | 2 +- .../patches/vendor/dparse-local-yaml.patch | 20 --- tests/integration/conftest.py | 6 +- 10 files changed, 198 insertions(+), 128 deletions(-) delete mode 100644 tasks/vendoring/patches/vendor/dparse-local-yaml.patch diff --git a/pipenv/vendor/dparse/__init__.py b/pipenv/vendor/dparse/__init__.py index afe8f63f..c4d4ee76 100644 --- a/pipenv/vendor/dparse/__init__.py +++ b/pipenv/vendor/dparse/__init__.py @@ -4,6 +4,6 @@ from __future__ import unicode_literals, absolute_import __author__ = """Jannis Gebauer""" __email__ = 'support@pyup.io' -__version__ = '0.5.1' +__version__ = '0.6.2' from .parser import parse # noqa diff --git a/pipenv/vendor/dparse/dependencies.py b/pipenv/vendor/dparse/dependencies.py index 8881a9bc..2bc63b0a 100644 --- a/pipenv/vendor/dparse/dependencies.py +++ b/pipenv/vendor/dparse/dependencies.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals, absolute_import import json +from json import JSONEncoder from . import filetypes, errors @@ -11,7 +12,9 @@ class Dependency(object): """ - def __init__(self, name, specs, line, source="pypi", meta={}, extras=[], line_numbers=None, index_server=None, hashes=(), dependency_type=None, section=None): + def __init__(self, name, specs, line, source="pypi", meta={}, extras=[], + line_numbers=None, index_server=None, hashes=(), + dependency_type=None, section=None): """ :param name: @@ -87,12 +90,25 @@ class Dependency(object): return self.name +class DparseJSONEncoder(JSONEncoder): + def default(self, o): + from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet + + if isinstance(o, SpecifierSet): + return str(o) + if isinstance(o, set): + return list(o) + + return JSONEncoder.default(self, o) + + class DependencyFile(object): """ """ - def __init__(self, content, path=None, sha=None, file_type=None, marker=((), ()), parser=None): + def __init__(self, content, path=None, sha=None, file_type=None, + marker=((), ()), parser=None, resolve=False): """ :param content: @@ -130,9 +146,11 @@ class DependencyFile(object): self.parser = parser_class.PipfileLockParser elif file_type == filetypes.setup_cfg: self.parser = parser_class.SetupCfgParser + elif file_type == filetypes.poetry_lock: + self.parser = parser_class.PoetryLockParser elif path is not None: - if path.endswith(".txt"): + if path.endswith((".txt", ".in")): self.parser = parser_class.RequirementsTXTParser elif path.endswith(".yml"): self.parser = parser_class.CondaYMLParser @@ -144,11 +162,23 @@ class DependencyFile(object): self.parser = parser_class.PipfileLockParser elif path.endswith("setup.cfg"): self.parser = parser_class.SetupCfgParser + elif path.endswith(filetypes.poetry_lock): + self.parser = parser_class.PoetryLockParser if not hasattr(self, "parser"): raise errors.UnknownDependencyFileError - self.parser = self.parser(self) + self.parser = self.parser(self, resolve=resolve) + + @property + def resolved_dependencies(self): + deps = self.dependencies.copy() + + for d in self.resolved_files: + if isinstance(d, DependencyFile): + deps.extend(d.resolved_dependencies) + + return deps def serialize(self): """ @@ -160,7 +190,9 @@ class DependencyFile(object): "content": self.content, "path": self.path, "sha": self.sha, - "dependencies": [dep.serialize() for dep in self.dependencies] + "dependencies": [dep.serialize() for dep in self.dependencies], + "resolved_dependencies": [dep.serialize() for dep in + self.resolved_dependencies] } @classmethod @@ -170,7 +202,8 @@ class DependencyFile(object): :param d: :return: """ - dependencies = [Dependency.deserialize(dep) for dep in d.pop("dependencies", [])] + dependencies = [Dependency.deserialize(dep) for dep in + d.pop("dependencies", [])] instance = cls(**d) instance.dependencies = dependencies return instance @@ -180,7 +213,7 @@ class DependencyFile(object): :return: """ - return json.dumps(self.serialize(), indent=2) + return json.dumps(self.serialize(), indent=2, cls=DparseJSONEncoder) def parse(self): """ @@ -192,5 +225,6 @@ class DependencyFile(object): return self self.parser.parse() - self.is_valid = len(self.dependencies) > 0 or len(self.resolved_files) > 0 + self.is_valid = len(self.dependencies) > 0 or len( + self.resolved_files) > 0 return self diff --git a/pipenv/vendor/dparse/errors.py b/pipenv/vendor/dparse/errors.py index b7a65430..74fc99db 100644 --- a/pipenv/vendor/dparse/errors.py +++ b/pipenv/vendor/dparse/errors.py @@ -1,8 +1,19 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, absolute_import + class UnknownDependencyFileError(Exception): """ """ - pass + def __init__(self, message="Unknown File type to parse"): + self.message = message + super().__init__(self.message) + + +class MalformedDependencyFileError(Exception): + + def __init__(self, message="The dependency file is malformed. {info}", + info=""): + self.message = message.format(info=info) + super().__init__(self.message) diff --git a/pipenv/vendor/dparse/filetypes.py b/pipenv/vendor/dparse/filetypes.py index df8a913e..eadfff45 100644 --- a/pipenv/vendor/dparse/filetypes.py +++ b/pipenv/vendor/dparse/filetypes.py @@ -7,3 +7,4 @@ setup_cfg = "setup.cfg" tox_ini = "tox.ini" pipfile = "Pipfile" pipfile_lock = "Pipfile.lock" +poetry_lock = "poetry.lock" diff --git a/pipenv/vendor/dparse/parser.py b/pipenv/vendor/dparse/parser.py index bae38b71..faaad2e8 100644 --- a/pipenv/vendor/dparse/parser.py +++ b/pipenv/vendor/dparse/parser.py @@ -1,20 +1,25 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, absolute_import + +import os from collections import OrderedDict import re from io import StringIO -from configparser import SafeConfigParser, NoOptionError +from configparser import ConfigParser, NoOptionError +from pathlib import PurePath - -from .regex import URL_REGEX, HASH_REGEX +from .errors import MalformedDependencyFileError +from .regex import HASH_REGEX from .dependencies import DependencyFile, Dependency -from pipenv.patched.pip._vendor.packaging.requirements import Requirement as PackagingRequirement, InvalidRequirement +from pipenv.patched.pip._vendor.packaging.requirements import Requirement as PackagingRequirement,\ + InvalidRequirement from . import filetypes import pipenv.vendor.toml as toml from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet +from pipenv.patched.pip._vendor.packaging.version import Version, InvalidVersion import json @@ -22,23 +27,24 @@ import json def setuptools_parse_requirements_backport(strs): # pragma: no cover # Copyright (C) 2016 Jason R Coombs # - # Permission is hereby granted, free of charge, to any person obtaining a copy of - # this software and associated documentation files (the "Software"), to deal in - # the Software without restriction, including without limitation the rights to - # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies - # of the Software, and to permit persons to whom the Software is furnished to do - # so, subject to the following conditions: + # Permission is hereby granted, free of charge, to any person obtaining a + # copy of this software and associated documentation files + # (the "Software"), to deal in the Software without restriction, including + # without limitation the rights to use, copy, modify, merge, publish, + # distribute, sublicense, and/or sell copies of the Software, and to permit + # persons to whom the Software is furnished to do so, subject to the + # following conditions: # - # The above copyright notice and this permission notice shall be included in all - # copies or substantial portions of the Software. + # The above copyright notice and this permission notice shall be included + # in all copies or substantial portions of the Software. # - # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - # SOFTWARE. + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """Yield ``Requirement`` objects for each specification in `strs` `strs` must be a string, or a (possibly-nested) iterable thereof. @@ -82,9 +88,11 @@ class RequirementsTXTLineParser(object): :return: """ try: - # setuptools requires a space before the comment. If this isn't the case, add it. + # setuptools requires a space before the comment. + # If this isn't the case, add it. if "\t#" in line: - parsed, = setuptools_parse_requirements_backport(line.replace("\t#", "\t #")) + parsed, = setuptools_parse_requirements_backport( + line.replace("\t#", "\t #")) else: parsed, = setuptools_parse_requirements_backport(line) except InvalidRequirement: @@ -104,13 +112,14 @@ class Parser(object): """ - def __init__(self, obj): + def __init__(self, obj, resolve=False): """ :param obj: """ self.obj = obj self._lines = None + self.resolve = resolve def iter_lines(self, lineno=0): """ @@ -175,10 +184,11 @@ class Parser(object): :param line: :return: """ - matches = URL_REGEX.findall(line) - if matches: - url = matches[0] - return url if url.endswith("/") else url + "/" + groups = re.split(pattern=r"[=\s]+", string=line.strip(), maxsplit=100) + + if len(groups) >= 2: + return groups[1] if groups[1].endswith("/") else groups[1] + "/" + return None @classmethod @@ -190,12 +200,10 @@ class Parser(object): :return: """ line = line.replace("-r ", "").replace("--requirement ", "") - parts = file_path.split("/") + normalized_path = PurePath(file_path) if " #" in line: line = line.split("#")[0].strip() - if len(parts) == 1: - return line - return "/".join(parts[:-1]) + "/" + line + return str(normalized_path.parent.joinpath(line)) class RequirementsTXTParser(Parser): @@ -216,17 +224,37 @@ class RequirementsTXTParser(Parser): # comments are lines that start with # only continue if line.startswith('-i') or \ - line.startswith('--index-url') or \ - line.startswith('--extra-index-url'): + line.startswith('--index-url') or \ + line.startswith('--extra-index-url'): # this file is using a private index server, try to parse it index_server = self.parse_index_server(line) continue - elif self.obj.path and (line.startswith('-r') or line.startswith('--requirement')): - self.obj.resolved_files.append(self.resolve_file(self.obj.path, line)) + elif self.obj.path and \ + (line.startswith('-r') or + line.startswith('--requirement')): + + req_file_path = self.resolve_file(self.obj.path, line) + + if self.resolve and os.path.exists(req_file_path): + with open(req_file_path, 'r') as f: + content = f.read() + + dep_file = DependencyFile( + content=content, + path=req_file_path, + resolve=True + ) + dep_file.parse() + self.obj.resolved_files.append(dep_file) + else: + self.obj.resolved_files.append(req_file_path) + elif line.startswith('-f') or line.startswith('--find-links') or \ - line.startswith('--no-index') or line.startswith('--allow-external') or \ - line.startswith('--allow-unverified') or line.startswith('-Z') or \ - line.startswith('--always-unzip'): + line.startswith('--no-index') or \ + line.startswith('--allow-external') or \ + line.startswith('--allow-unverified') or \ + line.startswith('-Z') or \ + line.startswith('--always-unzip'): continue elif self.is_marked_line(line): continue @@ -239,7 +267,8 @@ class RequirementsTXTParser(Parser): if "\\" in line: parseable_line = line.replace("\\", "") for next_line in self.iter_lines(num + 1): - parseable_line += next_line.strip().replace("\\", "") + parseable_line += next_line.strip().replace("\\", + "") line += "\n" + next_line if "\\" in next_line: continue @@ -250,7 +279,8 @@ class RequirementsTXTParser(Parser): hashes = [] if "--hash" in parseable_line: - parseable_line, hashes = Parser.parse_hashes(parseable_line) + parseable_line, hashes = Parser.parse_hashes( + parseable_line) req = RequirementsTXTLineParser.parse(parseable_line) if req: @@ -273,7 +303,7 @@ class ToxINIParser(Parser): :return: """ - parser = SafeConfigParser() + parser = ConfigParser() parser.readfp(StringIO(self.obj.content)) for section in parser.sections(): try: @@ -303,7 +333,8 @@ class CondaYMLParser(Parser): import yaml try: data = yaml.safe_load(self.obj.content) - if data and 'dependencies' in data and isinstance(data['dependencies'], list): + if data and 'dependencies' in data and \ + isinstance(data['dependencies'], list): for dep in data['dependencies']: if isinstance(dep, dict) and 'pip' in dep: for n, line in enumerate(dep['pip']): @@ -343,9 +374,10 @@ class PipfileParser(Parser): section=package_type ) ) - except (toml.TomlDecodeError, IndexError) as e: + except (toml.TomlDecodeError, IndexError): pass + class PipfileLockParser(Parser): def parse(self): @@ -373,13 +405,13 @@ class PipfileLockParser(Parser): section=package_type ) ) - except ValueError: - pass + except ValueError as e: + raise MalformedDependencyFileError(info=str(e)) class SetupCfgParser(Parser): def parse(self): - parser = SafeConfigParser() + parser = ConfigParser() parser.readfp(StringIO(self.obj.content)) for section in parser.values(): if section.name == 'options': @@ -404,7 +436,47 @@ class SetupCfgParser(Parser): self.obj.dependencies.append(req) -def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=None): +class PoetryLockParser(Parser): + + def parse(self): + """ + Parse a poetry.lock + """ + try: + data = toml.loads(self.obj.content, _dict=OrderedDict) + pkg_key = 'package' + if data: + try: + dependencies = data[pkg_key] + except KeyError: + raise KeyError( + "Poetry lock file is missing the package section") + + for dep in dependencies: + try: + name = dep['name'] + spec = "=={version}".format( + version=Version(dep['version'])) + section = dep['category'] + except KeyError: + raise KeyError("Malformed poetry lock file") + except InvalidVersion: + continue + + self.obj.dependencies.append( + Dependency( + name=name, specs=SpecifierSet(spec), + dependency_type=filetypes.poetry_lock, + line=''.join([name, spec]), + section=section + ) + ) + except (toml.TomlDecodeError, IndexError) as e: + raise MalformedDependencyFileError(info=str(e)) + + +def parse(content, file_type=None, path=None, sha=None, marker=((), ()), + parser=None, resolve=False): """ :param content: @@ -415,13 +487,15 @@ def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser= :param parser: :return: """ + dep_file = DependencyFile( content=content, path=path, sha=sha, marker=marker, file_type=file_type, - parser=parser + parser=parser, + resolve=resolve ) return dep_file.parse() diff --git a/pipenv/vendor/dparse/regex.py b/pipenv/vendor/dparse/regex.py index 40cc4091..4a1204e1 100644 --- a/pipenv/vendor/dparse/regex.py +++ b/pipenv/vendor/dparse/regex.py @@ -1,39 +1,4 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals -import re -# see https://gist.github.com/dperini/729294 -URL_REGEX = re.compile( - # protocol identifier - "(?:(?:https?|ftp)://)" - # user:pass authentication - "(?:\S+(?::\S*)?@)?" - "(?:" - # IP address exclusion - # private & local networks - "(?!(?:10|127)(?:\.\d{1,3}){3})" - "(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})" - "(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})" - # IP address dotted notation octets - # excludes loopback network 0.0.0.0 - # excludes reserved space >= 224.0.0.0 - # excludes network & broadcast addresses - # (first & last IP address of each class) - "(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])" - "(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}" - "(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))" - "|" - # host name - "(?:(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)" - # domain name - "(?:\.(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)*" - # TLD identifier - "(?:\.(?:[a-z\u00a1-\uffff]{2,}))" - ")" - # port number - "(?::\d{2,5})?" - # resource path - "(?:/\S*)?", - re.UNICODE) - -HASH_REGEX = r"--hash[=| ][\w]+:[\w]+" +HASH_REGEX = r"--hash[=| ]\w+:\w+" diff --git a/pipenv/vendor/dparse/updater.py b/pipenv/vendor/dparse/updater.py index 48117110..7b7ba9a5 100644 --- a/pipenv/vendor/dparse/updater.py +++ b/pipenv/vendor/dparse/updater.py @@ -8,24 +8,26 @@ import os class RequirementsTXTUpdater(object): - SUB_REGEX = r"^{}(?=\s*\r?\n?$)" @classmethod def update(cls, content, dependency, version, spec="==", hashes=()): """ - Updates the requirement to the latest version for the given content and adds hashes - if neccessary. + Updates the requirement to the latest version for the given content + and adds hashes if necessary. :param content: str, content :return: str, updated content """ - new_line = "{name}{spec}{version}".format(name=dependency.full_name, spec=spec, version=version) + new_line = "{name}{spec}{version}".format(name=dependency.full_name, + spec=spec, version=version) appendix = '' # leave environment markers intact if ";" in dependency.line: - # condense multiline, split out the env marker, strip comments and --hashes - new_line += ";" + dependency.line.splitlines()[0].split(";", 1)[1] \ - .split("#")[0].split("--hash")[0].rstrip() + # condense multiline, split out the env marker, strip comments + # and --hashes + new_line += ";" + \ + dependency.line.splitlines()[0].split(";", 1)[1] \ + .split("#")[0].split("--hash")[0].rstrip() # add the comment if "#" in dependency.line: # split the line into parts: requirement and comment @@ -40,7 +42,8 @@ class RequirementsTXTUpdater(object): else: break appendix += trailing_whitespace + "#" + comment - # if this is a hashed requirement, add a multiline break before the comment + # if this is a hashed requirement, add a multiline break before the + # comment if dependency.hashes and not new_line.endswith("\\"): new_line += " \\" # if this is a hashed requirement, add the hashes @@ -81,14 +84,16 @@ class PipfileUpdater(object): for package_type in ['packages', 'dev-packages']: if package_type in data: if dependency.full_name in data[package_type]: - data[package_type][dependency.full_name] = "{spec}{version}".format( + data[package_type][ + dependency.full_name] = "{spec}{version}".format( spec=spec, version=version ) try: from pipenv.project import Project except ImportError: - raise ImportError("Updating a Pipfile requires the pipenv extra to be installed. Install it with " - "pip install dparse[pipenv]") + raise ImportError( + "Updating a Pipfile requires the pipenv extra to be installed." + " Install it with pip install dparse[pipenv]") pipfile = tempfile.NamedTemporaryFile(delete=False) p = Project(chdir=False) p.write_toml(data=data, path=pipfile.name) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 0238665b..10f95dd4 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -5,7 +5,7 @@ click-didyoumean==0.0.3 click==8.0.3 colorama==0.4.4 distlib==0.3.2 -dparse==0.5.1 +dparse==0.6.2 idna==3.2 markupsafe==2.0.1 parse==1.19.0 diff --git a/tasks/vendoring/patches/vendor/dparse-local-yaml.patch b/tasks/vendoring/patches/vendor/dparse-local-yaml.patch deleted file mode 100644 index 896b9ee0..00000000 --- a/tasks/vendoring/patches/vendor/dparse-local-yaml.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/dparse/parser.py b/dparse/parser.py -index c01ebab..18689a5 100644 ---- a/pipenv/vendor/dparse/parser.py -+++ b/pipenv/vendor/dparse/parser.py -@@ -2,7 +2,6 @@ - from __future__ import unicode_literals, absolute_import - from collections import OrderedDict - import re --import yaml - - from io import StringIO - -@@ -301,6 +300,7 @@ class CondaYMLParser(Parser): - - :return: - """ -+ import yaml - try: - data = yaml.safe_load(self.obj.content) - if data and 'dependencies' in data and isinstance(data['dependencies'], list): diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d99afad8..efa3876d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -16,9 +16,6 @@ from tempfile import TemporaryDirectory import pytest import requests from click.testing import CliRunner -from pytest_pypi.app import prepare_fixtures -from pytest_pypi.app import prepare_packages as prepare_pypi_packages -import pypiserver from pipenv.cli import cli from pipenv.exceptions import VirtualenvActivationException @@ -31,6 +28,9 @@ from pipenv.vendor.vistir.path import ( create_tracked_tempdir, handle_remove_readonly, mkdir_p ) +from pytest_pypi.app import prepare_fixtures +from pytest_pypi.app import prepare_packages as prepare_pypi_packages +import pypiserver log = logging.getLogger(__name__) warnings.simplefilter("default", category=ResourceWarning)