Bump dparse version to 0.6.2

pipenv/vendor/dparse/__init__.py

@@ -4,6 +4,6 @@ from __future__ import unicode_literals, absolute_import
 
 __author__ = """Jannis Gebauer"""
 __email__ = 'support@pyup.io'
-__version__ = '0.5.1'
+__version__ = '0.6.2'
 
 from .parser import parse  # noqa

pipenv/vendor/dparse/dependencies.py

@@ -2,6 +2,7 @@
 from __future__ import unicode_literals, absolute_import
 
 import json
+from json import JSONEncoder
 
 from . import filetypes, errors
 
@@ -11,7 +12,9 @@ class Dependency(object):
 
     """
 
-    def __init__(self, name, specs, line, source="pypi", meta={}, extras=[], line_numbers=None, index_server=None, hashes=(), dependency_type=None, section=None):
+    def __init__(self, name, specs, line, source="pypi", meta={}, extras=[],
+                 line_numbers=None, index_server=None, hashes=(),
+                 dependency_type=None, section=None):
         """
 
         :param name:
@@ -87,12 +90,25 @@ class Dependency(object):
         return self.name
 
 
+class DparseJSONEncoder(JSONEncoder):
+    def default(self, o):
+        from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet
+
+        if isinstance(o, SpecifierSet):
+            return str(o)
+        if isinstance(o, set):
+            return list(o)
+
+        return JSONEncoder.default(self, o)
+
+
 class DependencyFile(object):
     """
 
     """
 
-    def __init__(self, content, path=None, sha=None, file_type=None, marker=((), ()), parser=None):
+    def __init__(self, content, path=None, sha=None, file_type=None,
+                 marker=((), ()), parser=None, resolve=False):
         """
 
         :param content:
@@ -130,9 +146,11 @@ class DependencyFile(object):
                     self.parser = parser_class.PipfileLockParser
                 elif file_type == filetypes.setup_cfg:
                     self.parser = parser_class.SetupCfgParser
+                elif file_type == filetypes.poetry_lock:
+                    self.parser = parser_class.PoetryLockParser
 
             elif path is not None:
-                if path.endswith(".txt"):
+                if path.endswith((".txt", ".in")):
                     self.parser = parser_class.RequirementsTXTParser
                 elif path.endswith(".yml"):
                     self.parser = parser_class.CondaYMLParser
@@ -144,11 +162,23 @@ class DependencyFile(object):
                     self.parser = parser_class.PipfileLockParser
                 elif path.endswith("setup.cfg"):
                     self.parser = parser_class.SetupCfgParser
+                elif path.endswith(filetypes.poetry_lock):
+                    self.parser = parser_class.PoetryLockParser
 
         if not hasattr(self, "parser"):
             raise errors.UnknownDependencyFileError
 
-        self.parser = self.parser(self)
+        self.parser = self.parser(self, resolve=resolve)
+
+    @property
+    def resolved_dependencies(self):
+        deps = self.dependencies.copy()
+
+        for d in self.resolved_files:
+            if isinstance(d, DependencyFile):
+                deps.extend(d.resolved_dependencies)
+
+        return deps
 
     def serialize(self):
         """
@@ -160,7 +190,9 @@ class DependencyFile(object):
             "content": self.content,
             "path": self.path,
             "sha": self.sha,
-            "dependencies": [dep.serialize() for dep in self.dependencies]
+            "dependencies": [dep.serialize() for dep in self.dependencies],
+            "resolved_dependencies": [dep.serialize() for dep in
+                                      self.resolved_dependencies]
         }
 
     @classmethod
@@ -170,7 +202,8 @@ class DependencyFile(object):
         :param d:
         :return:
         """
-        dependencies = [Dependency.deserialize(dep) for dep in d.pop("dependencies", [])]
+        dependencies = [Dependency.deserialize(dep) for dep in
+                        d.pop("dependencies", [])]
         instance = cls(**d)
         instance.dependencies = dependencies
         return instance
@@ -180,7 +213,7 @@ class DependencyFile(object):
 
         :return:
         """
-        return json.dumps(self.serialize(), indent=2)
+        return json.dumps(self.serialize(), indent=2, cls=DparseJSONEncoder)
 
     def parse(self):
         """
@@ -192,5 +225,6 @@ class DependencyFile(object):
             return self
         self.parser.parse()
 
-        self.is_valid = len(self.dependencies) > 0 or len(self.resolved_files) > 0
+        self.is_valid = len(self.dependencies) > 0 or len(
+            self.resolved_files) > 0
         return self

pipenv/vendor/dparse/errors.py

@@ -1,8 +1,19 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, absolute_import
 
+
 class UnknownDependencyFileError(Exception):
     """
 
     """
-    pass
+    def __init__(self, message="Unknown File type to parse"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class MalformedDependencyFileError(Exception):
+
+    def __init__(self, message="The dependency file is malformed. {info}",
+                 info=""):
+        self.message = message.format(info=info)
+        super().__init__(self.message)

pipenv/vendor/dparse/filetypes.py

@@ -7,3 +7,4 @@ setup_cfg = "setup.cfg"
 tox_ini = "tox.ini"
 pipfile = "Pipfile"
 pipfile_lock = "Pipfile.lock"
+poetry_lock = "poetry.lock"

pipenv/vendor/dparse/parser.py

@@ -1,20 +1,25 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, absolute_import
+
+import os
 from collections import OrderedDict
 import re
 
 from io import StringIO
 
-from configparser import SafeConfigParser, NoOptionError
+from configparser import ConfigParser, NoOptionError
+from pathlib import PurePath
 
-
-from .regex import URL_REGEX, HASH_REGEX
+from .errors import MalformedDependencyFileError
+from .regex import HASH_REGEX
 
 from .dependencies import DependencyFile, Dependency
-from pipenv.patched.pip._vendor.packaging.requirements import Requirement as PackagingRequirement, InvalidRequirement
+from pipenv.patched.pip._vendor.packaging.requirements import Requirement as PackagingRequirement,\
+    InvalidRequirement
 from . import filetypes
 import pipenv.vendor.toml as toml
 from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet
+from pipenv.patched.pip._vendor.packaging.version import Version, InvalidVersion
 import json
 
 
@@ -22,23 +27,24 @@ import json
 def setuptools_parse_requirements_backport(strs):  # pragma: no cover
     # Copyright (C) 2016 Jason R Coombs <jaraco@jaraco.com>
     #
-    # Permission is hereby granted, free of charge, to any person obtaining a copy of
-    # this software and associated documentation files (the "Software"), to deal in
-    # the Software without restriction, including without limitation the rights to
-    # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-    # of the Software, and to permit persons to whom the Software is furnished to do
-    # so, subject to the following conditions:
+    # Permission is hereby granted, free of charge, to any person obtaining a
+    # copy of this software and associated documentation files
+    # (the "Software"), to deal in the Software without restriction, including
+    # without limitation the rights to use, copy, modify, merge, publish,
+    # distribute, sublicense, and/or sell copies of the Software, and to permit
+    # persons to whom the Software is furnished to do so, subject to the
+    # following conditions:
     #
-    # The above copyright notice and this permission notice shall be included in all
-    # copies or substantial portions of the Software.
+    # The above copyright notice and this permission notice shall be included
+    # in all copies or substantial portions of the Software.
     #
-    # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-    # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-    # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-    # SOFTWARE.
+    # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+    # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+    # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+    # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+    # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+    # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
     """Yield ``Requirement`` objects for each specification in `strs`
 
     `strs` must be a string, or a (possibly-nested) iterable thereof.
@@ -82,9 +88,11 @@ class RequirementsTXTLineParser(object):
         :return:
         """
         try:
-            # setuptools requires a space before the comment. If this isn't the case, add it.
+            # setuptools requires a space before the comment.
+            # If this isn't the case, add it.
             if "\t#" in line:
-                parsed, = setuptools_parse_requirements_backport(line.replace("\t#", "\t #"))
+                parsed, = setuptools_parse_requirements_backport(
+                    line.replace("\t#", "\t #"))
             else:
                 parsed, = setuptools_parse_requirements_backport(line)
         except InvalidRequirement:
@@ -104,13 +112,14 @@ class Parser(object):
 
     """
 
-    def __init__(self, obj):
+    def __init__(self, obj, resolve=False):
         """
 
         :param obj:
         """
         self.obj = obj
         self._lines = None
+        self.resolve = resolve
 
     def iter_lines(self, lineno=0):
         """
@@ -175,10 +184,11 @@ class Parser(object):
         :param line:
         :return:
         """
-        matches = URL_REGEX.findall(line)
-        if matches:
-            url = matches[0]
-            return url if url.endswith("/") else url + "/"
+        groups = re.split(pattern=r"[=\s]+", string=line.strip(), maxsplit=100)
+
+        if len(groups) >= 2:
+            return groups[1] if groups[1].endswith("/") else groups[1] + "/"
+
         return None
 
     @classmethod
@@ -190,12 +200,10 @@ class Parser(object):
         :return:
         """
         line = line.replace("-r ", "").replace("--requirement ", "")
-        parts = file_path.split("/")
+        normalized_path = PurePath(file_path)
         if " #" in line:
             line = line.split("#")[0].strip()
-        if len(parts) == 1:
-            return line
-        return "/".join(parts[:-1]) + "/" + line
+        return str(normalized_path.parent.joinpath(line))
 
 
 class RequirementsTXTParser(Parser):
@@ -216,17 +224,37 @@ class RequirementsTXTParser(Parser):
                 # comments are lines that start with # only
                 continue
             if line.startswith('-i') or \
-                line.startswith('--index-url') or \
-                line.startswith('--extra-index-url'):
+               line.startswith('--index-url') or \
+               line.startswith('--extra-index-url'):
                 # this file is using a private index server, try to parse it
                 index_server = self.parse_index_server(line)
                 continue
-            elif self.obj.path and (line.startswith('-r') or line.startswith('--requirement')):
-                self.obj.resolved_files.append(self.resolve_file(self.obj.path, line))
+            elif self.obj.path and \
+                    (line.startswith('-r') or
+                     line.startswith('--requirement')):
+
+                req_file_path = self.resolve_file(self.obj.path, line)
+
+                if self.resolve and os.path.exists(req_file_path):
+                    with open(req_file_path, 'r') as f:
+                        content = f.read()
+
+                        dep_file = DependencyFile(
+                            content=content,
+                            path=req_file_path,
+                            resolve=True
+                        )
+                        dep_file.parse()
+                        self.obj.resolved_files.append(dep_file)
+                else:
+                    self.obj.resolved_files.append(req_file_path)
+
             elif line.startswith('-f') or line.startswith('--find-links') or \
-                line.startswith('--no-index') or line.startswith('--allow-external') or \
-                line.startswith('--allow-unverified') or line.startswith('-Z') or \
-                line.startswith('--always-unzip'):
+                line.startswith('--no-index') or \
+                line.startswith('--allow-external') or \
+                line.startswith('--allow-unverified') or \
+                line.startswith('-Z') or \
+                    line.startswith('--always-unzip'):
                 continue
             elif self.is_marked_line(line):
                 continue
@@ -239,7 +267,8 @@ class RequirementsTXTParser(Parser):
                     if "\\" in line:
                         parseable_line = line.replace("\\", "")
                         for next_line in self.iter_lines(num + 1):
-                            parseable_line += next_line.strip().replace("\\", "")
+                            parseable_line += next_line.strip().replace("\\",
+                                                                        "")
                             line += "\n" + next_line
                             if "\\" in next_line:
                                 continue
@@ -250,7 +279,8 @@ class RequirementsTXTParser(Parser):
 
                     hashes = []
                     if "--hash" in parseable_line:
-                        parseable_line, hashes = Parser.parse_hashes(parseable_line)
+                        parseable_line, hashes = Parser.parse_hashes(
+                            parseable_line)
 
                     req = RequirementsTXTLineParser.parse(parseable_line)
                     if req:
@@ -273,7 +303,7 @@ class ToxINIParser(Parser):
 
         :return:
         """
-        parser = SafeConfigParser()
+        parser = ConfigParser()
         parser.readfp(StringIO(self.obj.content))
         for section in parser.sections():
             try:
@@ -303,7 +333,8 @@ class CondaYMLParser(Parser):
         import yaml
         try:
             data = yaml.safe_load(self.obj.content)
-            if data and 'dependencies' in data and isinstance(data['dependencies'], list):
+            if data and 'dependencies' in data and \
+                    isinstance(data['dependencies'], list):
                 for dep in data['dependencies']:
                     if isinstance(dep, dict) and 'pip' in dep:
                         for n, line in enumerate(dep['pip']):
@@ -343,9 +374,10 @@ class PipfileParser(Parser):
                                     section=package_type
                                 )
                             )
-        except (toml.TomlDecodeError, IndexError) as e:
+        except (toml.TomlDecodeError, IndexError):
             pass
 
+
 class PipfileLockParser(Parser):
 
     def parse(self):
@@ -373,13 +405,13 @@ class PipfileLockParser(Parser):
                                     section=package_type
                                 )
                             )
-        except ValueError:
-            pass
+        except ValueError as e:
+            raise MalformedDependencyFileError(info=str(e))
 
 
 class SetupCfgParser(Parser):
     def parse(self):
-        parser = SafeConfigParser()
+        parser = ConfigParser()
         parser.readfp(StringIO(self.obj.content))
         for section in parser.values():
             if section.name == 'options':
@@ -404,7 +436,47 @@ class SetupCfgParser(Parser):
                     self.obj.dependencies.append(req)
 
 
-def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=None):
+class PoetryLockParser(Parser):
+
+    def parse(self):
+        """
+        Parse a poetry.lock
+        """
+        try:
+            data = toml.loads(self.obj.content, _dict=OrderedDict)
+            pkg_key = 'package'
+            if data:
+                try:
+                    dependencies = data[pkg_key]
+                except KeyError:
+                    raise KeyError(
+                        "Poetry lock file is missing the package section")
+
+                for dep in dependencies:
+                    try:
+                        name = dep['name']
+                        spec = "=={version}".format(
+                            version=Version(dep['version']))
+                        section = dep['category']
+                    except KeyError:
+                        raise KeyError("Malformed poetry lock file")
+                    except InvalidVersion:
+                        continue
+
+                    self.obj.dependencies.append(
+                        Dependency(
+                            name=name, specs=SpecifierSet(spec),
+                            dependency_type=filetypes.poetry_lock,
+                            line=''.join([name, spec]),
+                            section=section
+                        )
+                    )
+        except (toml.TomlDecodeError, IndexError) as e:
+            raise MalformedDependencyFileError(info=str(e))
+
+
+def parse(content, file_type=None, path=None, sha=None, marker=((), ()),
+          parser=None, resolve=False):
     """
 
     :param content:
@@ -415,13 +487,15 @@ def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=
     :param parser:
     :return:
     """
+
     dep_file = DependencyFile(
         content=content,
         path=path,
         sha=sha,
         marker=marker,
         file_type=file_type,
-        parser=parser
+        parser=parser,
+        resolve=resolve
     )
 
     return dep_file.parse()

pipenv/vendor/dparse/regex.py

@@ -1,39 +1,4 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, unicode_literals
 
-import re
-# see https://gist.github.com/dperini/729294
-URL_REGEX = re.compile(
-    # protocol identifier
-    "(?:(?:https?|ftp)://)"
-    # user:pass authentication
-    "(?:\S+(?::\S*)?@)?"
-    "(?:"
-    # IP address exclusion
-    # private & local networks
-    "(?!(?:10|127)(?:\.\d{1,3}){3})"
-    "(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})"
-    "(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})"
-    # IP address dotted notation octets
-    # excludes loopback network 0.0.0.0
-    # excludes reserved space >= 224.0.0.0
-    # excludes network & broadcast addresses
-    # (first & last IP address of each class)
-    "(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])"
-    "(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}"
-    "(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))"
-    "|"
-    # host name
-    "(?:(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)"
-    # domain name
-    "(?:\.(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)*"
-    # TLD identifier
-    "(?:\.(?:[a-z\u00a1-\uffff]{2,}))"
-    ")"
-    # port number
-    "(?::\d{2,5})?"
-    # resource path
-    "(?:/\S*)?",
-    re.UNICODE)
-
-HASH_REGEX = r"--hash[=| ][\w]+:[\w]+"
+HASH_REGEX = r"--hash[=| ]\w+:\w+"

pipenv/vendor/dparse/updater.py

@@ -8,24 +8,26 @@ import os
 
 
 class RequirementsTXTUpdater(object):
-
     SUB_REGEX = r"^{}(?=\s*\r?\n?$)"
 
     @classmethod
     def update(cls, content, dependency, version, spec="==", hashes=()):
         """
-        Updates the requirement to the latest version for the given content and adds hashes
-        if neccessary.
+        Updates the requirement to the latest version for the given content
+        and adds hashes if necessary.
         :param content: str, content
         :return: str, updated content
         """
-        new_line = "{name}{spec}{version}".format(name=dependency.full_name, spec=spec, version=version)
+        new_line = "{name}{spec}{version}".format(name=dependency.full_name,
+                                                  spec=spec, version=version)
         appendix = ''
         # leave environment markers intact
         if ";" in dependency.line:
-            # condense multiline, split out the env marker, strip comments and --hashes
-            new_line += ";" + dependency.line.splitlines()[0].split(";", 1)[1] \
-                .split("#")[0].split("--hash")[0].rstrip()
+            # condense multiline, split out the env marker, strip comments
+            # and --hashes
+            new_line += ";" + \
+                        dependency.line.splitlines()[0].split(";", 1)[1] \
+                            .split("#")[0].split("--hash")[0].rstrip()
         # add the comment
         if "#" in dependency.line:
             # split the line into parts: requirement and comment
@@ -40,7 +42,8 @@ class RequirementsTXTUpdater(object):
                 else:
                     break
             appendix += trailing_whitespace + "#" + comment
-        # if this is a hashed requirement, add a multiline break before the comment
+        # if this is a hashed requirement, add a multiline break before the
+        # comment
         if dependency.hashes and not new_line.endswith("\\"):
             new_line += " \\"
         # if this is a hashed requirement, add the hashes
@@ -81,14 +84,16 @@ class PipfileUpdater(object):
             for package_type in ['packages', 'dev-packages']:
                 if package_type in data:
                     if dependency.full_name in data[package_type]:
-                        data[package_type][dependency.full_name] = "{spec}{version}".format(
+                        data[package_type][
+                            dependency.full_name] = "{spec}{version}".format(
                             spec=spec, version=version
                         )
         try:
             from pipenv.project import Project
         except ImportError:
-            raise ImportError("Updating a Pipfile requires the pipenv extra to be installed. Install it with "
-                              "pip install dparse[pipenv]")
+            raise ImportError(
+                "Updating a Pipfile requires the pipenv extra to be installed."
+                " Install it with pip install dparse[pipenv]")
         pipfile = tempfile.NamedTemporaryFile(delete=False)
         p = Project(chdir=False)
         p.write_toml(data=data, path=pipfile.name)

pipenv/vendor/vendor.txt

@@ -5,7 +5,7 @@ click-didyoumean==0.0.3
 click==8.0.3
 colorama==0.4.4
 distlib==0.3.2
-dparse==0.5.1
+dparse==0.6.2
 idna==3.2
 markupsafe==2.0.1
 parse==1.19.0

tasks/vendoring/patches/vendor/dparse-local-yaml.patch

@@ -1,20 +0,0 @@
-diff --git a/dparse/parser.py b/dparse/parser.py
-index c01ebab..18689a5 100644
---- a/pipenv/vendor/dparse/parser.py
-+++ b/pipenv/vendor/dparse/parser.py
-@@ -2,7 +2,6 @@
- from __future__ import unicode_literals, absolute_import
- from collections import OrderedDict
- import re
--import yaml
- 
- from io import StringIO
- 
-@@ -301,6 +300,7 @@ class CondaYMLParser(Parser):
- 
-         :return:
-         """
-+        import yaml
-         try:
-             data = yaml.safe_load(self.obj.content)
-             if data and 'dependencies' in data and isinstance(data['dependencies'], list):

tests/integration/conftest.py

@@ -16,9 +16,6 @@ from tempfile import TemporaryDirectory
 import pytest
 import requests
 from click.testing import CliRunner
-from pytest_pypi.app import prepare_fixtures
-from pytest_pypi.app import prepare_packages as prepare_pypi_packages
-import pypiserver
 
 from pipenv.cli import cli
 from pipenv.exceptions import VirtualenvActivationException
@@ -31,6 +28,9 @@ from pipenv.vendor.vistir.path import (
     create_tracked_tempdir, handle_remove_readonly, mkdir_p
 )
 
+from pytest_pypi.app import prepare_fixtures
+from pytest_pypi.app import prepare_packages as prepare_pypi_packages
+import pypiserver
 
 log = logging.getLogger(__name__)
 warnings.simplefilter("default", category=ResourceWarning)