diff --git a/Pipfile.lock b/Pipfile.lock index dfdf6a35..5d58f7dd 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -77,11 +77,11 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a", - "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887", - "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae" + "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8", + "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368", + "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0" ], - "version": "==4.8.2" + "version": "==4.9.0" }, "black": { "hashes": [ @@ -107,10 +107,10 @@ }, "certifi": { "hashes": [ - "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", - "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" + "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", + "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" ], - "version": "==2019.11.28" + "version": "==2020.4.5.1" }, "cffi": { "hashes": [ @@ -289,11 +289,11 @@ }, "flask": { "hashes": [ - "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", - "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6" + "sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060", + "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.1.1" + "version": "==1.1.2" }, "funcsigs": { "hashes": [ @@ -391,19 +391,19 @@ }, "jedi": { "hashes": [ - "sha256:b4f4052551025c6b0b0b193b29a6ff7bdb74c52450631206c262aef9f7159ad2", - "sha256:d5c871cb9360b414f981e7072c52c33258d598305280fef91c6cae34739d65d5" + "sha256:cd60c93b71944d628ccac47df9a60fec53150de53d42dc10a7fc4b5ba6aae798", + "sha256:df40c97641cb943661d2db4c33c2e1ff75d491189423249e989bcea4464f3030" ], "index": "pypi", - "version": "==0.16.0" + "version": "==0.17.0" }, "jinja2": { "hashes": [ - "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250", - "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49" + "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", + "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.11.1" + "version": "==2.11.2" }, "markupsafe": { "hashes": [ @@ -484,11 +484,11 @@ }, "parso": { "hashes": [ - "sha256:0c5659e0c6eba20636f99a04f469798dca8da279645ce5c387315b2c23912157", - "sha256:8515fc12cfca6ee3aa59138741fc5624d62340c97e401c74875769948d4f2995" + "sha256:158c140fc04112dc45bca311633ae5033c2c2a7b732fa33d0955bad8152a8dd0", + "sha256:908e9fae2144a076d72ae4e25539143d40b8e3eafbaeae03c1bfe226f4cdf12c" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.6.2" + "version": "==0.7.0" }, "parver": { "hashes": [ @@ -520,10 +520,10 @@ }, "pbr": { "hashes": [ - "sha256:139d2625547dbfa5fb0b81daebb39601c478c21956dc57e2e07b74450a8c506b", - "sha256:61aa52a0f18b71c5cc58232d2cf8f8d09cd67fcad60b742a60124cb8d6951488" + "sha256:07f558fece33b05caf857474a366dfcc00562bca13dd8b47b2b3e22d9f9bf55c", + "sha256:579170e23f8e0c2f24b0de612f71f648eccb79fb1322c814ae6b3c07b5ba23e8" ], - "version": "==5.4.4" + "version": "==5.4.5" }, "pipenv": { "editable": true, @@ -590,11 +590,11 @@ }, "pyparsing": { "hashes": [ - "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", - "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b", + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.4.6" + "version": "==2.4.7" }, "pytest": { "hashes": [ @@ -759,11 +759,11 @@ }, "sphinx-click": { "hashes": [ - "sha256:793c68b41c4a9435f953e2a27f9bf5883729037b7431f32b2776257c2966bd1b", - "sha256:8c6274666730686a65efbae0b4465879b030372333de3114aeb63c44204da32e" + "sha256:06952d5de6cbe2cb7d6dc656bc471652d2b484cf1e1b2d65edb7f4f2e867c7f6", + "sha256:1b649ebe9f7a85b78ef6545d1dc258da5abca850ac6375be104d484a6334a728" ], "index": "pypi", - "version": "==2.3.1" + "version": "==2.3.2" }, "sphinxcontrib-websupport": { "hashes": [ @@ -803,11 +803,11 @@ }, "tqdm": { "hashes": [ - "sha256:03d2366c64d44c7f61e74c700d9b202d57e9efe355ea5c28814c52bfe7a50b8c", - "sha256:be5ddeec77d78ba781ea41eacb2358a77f74cc2407f54b82222d7ee7dc8c8ccf" + "sha256:00339634a22c10a7a22476ee946bbde2dbe48d042ded784e4d88e0236eca5d81", + "sha256:ea9e3fd6bd9a37e8783d75bfc4c1faf3c6813da6bd1c3e776488b41ec683af94" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==4.44.1" + "version": "==4.45.0" }, "twine": { "hashes": [ @@ -855,11 +855,11 @@ }, "urllib3": { "hashes": [ - "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", - "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" + "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115", + "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'", - "version": "==1.25.8" + "version": "==1.25.9" }, "virtualenv": { "hashes": [ diff --git a/news/4188.bugfix.rst b/news/4188.bugfix.rst new file mode 100644 index 00000000..0ea2c943 --- /dev/null +++ b/news/4188.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue with ``pipenv check`` failing due to an invalid API key from ``pyup.io``. diff --git a/news/4188.vendor.rst b/news/4188.vendor.rst new file mode 100644 index 00000000..e24a45fd --- /dev/null +++ b/news/4188.vendor.rst @@ -0,0 +1,11 @@ +Add and update vendored dependencies to accommodate ``safety`` vendoring: +- **safety** ``(none)`` => ``1.8.7`` +- **dparse** ``(none)`` => ``0.5.0`` +- **pyyaml** ``(none)`` => ``5.3.1`` +- **urllib3** ``1.25.8`` => ``1.25.9`` +- **certifi** ``2019.11.28`` => ``2020.4.5.1`` +- **pyparsing** ``2.4.6`` => ``2.4.7`` +- **resolvelib** ``0.2.2`` => ``0.3.0`` +- **importlib-metadata** ``1.5.1`` => ``1.6.0`` +- **pip-shims** ``0.5.1`` => ``0.5.2`` +- **requirementslib** ``1.5.5`` => ``1.5.6`` diff --git a/pipenv/core.py b/pipenv/core.py index 4810057d..2290d977 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -125,8 +125,10 @@ def do_clear(): from pip import locations try: - vistir.path.rmtree(PIPENV_CACHE_DIR) - vistir.path.rmtree(locations.USER_CACHE_DIR) + vistir.path.rmtree(PIPENV_CACHE_DIR, onerror=vistir.path.handle_remove_readonly) + vistir.path.rmtree( + locations.USER_CACHE_DIR, onerror=vistir.path.handle_remove_readonly + ) except OSError as e: # Ignore FileNotFoundError. This is needed for Python 2.7. import errno @@ -756,6 +758,9 @@ def batch_install(deps_list, procs, failed_deps_queue, del os.environ["PYTHONHOME"] if "GIT_CONFIG" in os.environ and dep.is_vcs: del os.environ["GIT_CONFIG"] + use_pep517 = True + if failed and not dep.is_vcs: + use_pep517 = False c = pip_install( dep, @@ -768,7 +773,7 @@ def batch_install(deps_list, procs, failed_deps_queue, pypi_mirror=pypi_mirror, trusted_hosts=trusted_hosts, extra_indexes=extra_indexes, - use_pep517=not failed, + use_pep517=use_pep517, ) c.dep = dep # if dep.is_vcs or dep.editable: @@ -2585,7 +2590,7 @@ def do_check( click.echo(crayons.normal(decode_for_output("Checking PEP 508 requirements…"), bold=True)) pep508checker_path = pep508checker.__file__.rstrip("cdo") safety_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "patched", "safety.zip" + os.path.dirname(os.path.abspath(__file__)), "patched", "safety" ) if not system: python = which("python") @@ -2646,9 +2651,10 @@ def do_check( err=True, ) else: - ignored = "" - key = "--key={0}".format(PIPENV_PYUP_API_KEY) - cmd = _cmd + [safety_path, "check", "--json", key] + ignored = [] + cmd = _cmd + [safety_path, "check", "--json"] + if PIPENV_PYUP_API_KEY: + cmd = cmd + ["--key={0}".format(PIPENV_PYUP_API_KEY)] if ignored: for cve in ignored: cmd += cve diff --git a/pipenv/environment.py b/pipenv/environment.py index 65e7188d..7538ea9e 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -90,7 +90,8 @@ class Environment(object): deps.add(dist) try: reqs = dist.requires() - except (AttributeError, OSError, IOError): # The METADATA file can't be found + # KeyError = limited metadata can be found + except (KeyError, AttributeError, OSError, IOError): # The METADATA file can't be found return deps for req in reqs: dist = working_set.find(req) diff --git a/pipenv/environments.py b/pipenv/environments.py index 622b76ff..79f66019 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -256,7 +256,7 @@ approach, you may set this to '0', 'off', or 'false'. """ PIPENV_PYUP_API_KEY = os.environ.get( - "PIPENV_PYUP_API_KEY", "1ab8d58f-5122e025-83674263-bc1e79e0" + "PIPENV_PYUP_API_KEY", None ) # Internal, support running in a different Python from sys.executable. diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index ce803faf..b8a2816a 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -1,4 +1,4 @@ -safety +safety==1.8.7 crayons==0.1.2 pipfile==0.0.2 pip-tools==4.3.0 diff --git a/pipenv/patched/safety/LICENSE b/pipenv/patched/safety.LICENSE similarity index 99% rename from pipenv/patched/safety/LICENSE rename to pipenv/patched/safety.LICENSE index 55a1eb03..c5fda558 100644 --- a/pipenv/patched/safety/LICENSE +++ b/pipenv/patched/safety.LICENSE @@ -1,4 +1,3 @@ - MIT License Copyright (c) 2016, pyup.io diff --git a/pipenv/patched/safety.zip b/pipenv/patched/safety.zip deleted file mode 100644 index b839971a..00000000 Binary files a/pipenv/patched/safety.zip and /dev/null differ diff --git a/pipenv/patched/safety/__main__.py b/pipenv/patched/safety/__main__.py index d9a0bdab..f905408a 100644 --- a/pipenv/patched/safety/__main__.py +++ b/pipenv/patched/safety/__main__.py @@ -1,8 +1,51 @@ """Allow safety to be executable through `python -m safety`.""" from __future__ import absolute_import -from .cli import cli +import os +import sys +import sysconfig + + +PATCHED_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PIPENV_DIR = os.path.dirname(PATCHED_DIR) +VENDORED_DIR = os.path.join("PIPENV_DIR", "vendor") + + +def get_site_packages(): + prefixes = {sys.prefix, sysconfig.get_config_var('prefix')} + try: + prefixes.add(sys.real_prefix) + except AttributeError: + pass + form = sysconfig.get_path('purelib', expand=False) + py_version_short = '{0[0]}.{0[1]}'.format(sys.version_info) + return { + form.format(base=prefix, py_version_short=py_version_short) + for prefix in prefixes + } + + +def insert_before_site_packages(*paths): + site_packages = get_site_packages() + index = None + for i, path in enumerate(sys.path): + if path in site_packages: + index = i + break + if index is None: + sys.path += list(paths) + else: + sys.path = sys.path[:index] + list(paths) + sys.path[index:] + + +def insert_pipenv_dirs(): + insert_before_site_packages(os.path.dirname(PIPENV_DIR), PATCHED_DIR, VENDORED_DIR) if __name__ == "__main__": # pragma: no cover + insert_pipenv_dirs() + yaml_lib = "pipenv.patched.yaml{0}".format(sys.version_info[0]) + locals()[yaml_lib] = __import__(yaml_lib) + sys.modules["yaml"] = sys.modules[yaml_lib] + from safety.cli import cli cli(prog_name="safety") diff --git a/pipenv/patched/yaml2/LICENSE b/pipenv/patched/yaml2/LICENSE new file mode 100644 index 00000000..3d82c281 --- /dev/null +++ b/pipenv/patched/yaml2/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2017-2020 Ingy döt Net +Copyright (c) 2006-2016 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/patched/yaml2/__init__.py b/pipenv/patched/yaml2/__init__.py new file mode 100644 index 00000000..211fc866 --- /dev/null +++ b/pipenv/patched/yaml2/__init__.py @@ -0,0 +1,431 @@ + +from error import * + +from tokens import * +from events import * +from nodes import * + +from loader import * +from dumper import * + +__version__ = '5.3.1' + +try: + from cyaml import * + __with_libyaml__ = True +except ImportError: + __with_libyaml__ = False + + +#------------------------------------------------------------------------------ +# Warnings control +#------------------------------------------------------------------------------ + +# 'Global' warnings state: +_warnings_enabled = { + 'YAMLLoadWarning': True, +} + +# Get or set global warnings' state +def warnings(settings=None): + if settings is None: + return _warnings_enabled + + if type(settings) is dict: + for key in settings: + if key in _warnings_enabled: + _warnings_enabled[key] = settings[key] + +# Warn when load() is called without Loader=... +class YAMLLoadWarning(RuntimeWarning): + pass + +def load_warning(method): + if _warnings_enabled['YAMLLoadWarning'] is False: + return + + import warnings + + message = ( + "calling yaml.%s() without Loader=... is deprecated, as the " + "default Loader is unsafe. Please read " + "https://msg.pyyaml.org/load for full details." + ) % method + + warnings.warn(message, YAMLLoadWarning, stacklevel=3) + +#------------------------------------------------------------------------------ +def scan(stream, Loader=Loader): + """ + Scan a YAML stream and produce scanning tokens. + """ + loader = Loader(stream) + try: + while loader.check_token(): + yield loader.get_token() + finally: + loader.dispose() + +def parse(stream, Loader=Loader): + """ + Parse a YAML stream and produce parsing events. + """ + loader = Loader(stream) + try: + while loader.check_event(): + yield loader.get_event() + finally: + loader.dispose() + +def compose(stream, Loader=Loader): + """ + Parse the first YAML document in a stream + and produce the corresponding representation tree. + """ + loader = Loader(stream) + try: + return loader.get_single_node() + finally: + loader.dispose() + +def compose_all(stream, Loader=Loader): + """ + Parse all YAML documents in a stream + and produce corresponding representation trees. + """ + loader = Loader(stream) + try: + while loader.check_node(): + yield loader.get_node() + finally: + loader.dispose() + +def load(stream, Loader=None): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + """ + if Loader is None: + load_warning('load') + Loader = FullLoader + + loader = Loader(stream) + try: + return loader.get_single_data() + finally: + loader.dispose() + +def load_all(stream, Loader=None): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + """ + if Loader is None: + load_warning('load_all') + Loader = FullLoader + + loader = Loader(stream) + try: + while loader.check_data(): + yield loader.get_data() + finally: + loader.dispose() + +def full_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load(stream, FullLoader) + +def full_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load_all(stream, FullLoader) + +def safe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. + """ + return load(stream, SafeLoader) + +def safe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. + """ + return load_all(stream, SafeLoader) + +def unsafe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load(stream, UnsafeLoader) + +def unsafe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load_all(stream, UnsafeLoader) + +def emit(events, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + """ + Emit YAML parsing events into a stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + from StringIO import StringIO + stream = StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + try: + for event in events: + dumper.emit(event) + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def serialize_all(nodes, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding='utf-8', explicit_start=None, explicit_end=None, + version=None, tags=None): + """ + Serialize a sequence of representation trees into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + if encoding is None: + from StringIO import StringIO + else: + from cStringIO import StringIO + stream = StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end) + try: + dumper.open() + for node in nodes: + dumper.serialize(node) + dumper.close() + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def serialize(node, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a representation tree into a YAML stream. + If stream is None, return the produced string instead. + """ + return serialize_all([node], stream, Dumper=Dumper, **kwds) + +def dump_all(documents, stream=None, Dumper=Dumper, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding='utf-8', explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + """ + Serialize a sequence of Python objects into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + if encoding is None: + from StringIO import StringIO + else: + from cStringIO import StringIO + stream = StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, default_style=default_style, + default_flow_style=default_flow_style, + canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys) + try: + dumper.open() + for data in documents: + dumper.represent(data) + dumper.close() + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def dump(data, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a Python object into a YAML stream. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=Dumper, **kwds) + +def safe_dump_all(documents, stream=None, **kwds): + """ + Serialize a sequence of Python objects into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all(documents, stream, Dumper=SafeDumper, **kwds) + +def safe_dump(data, stream=None, **kwds): + """ + Serialize a Python object into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=SafeDumper, **kwds) + +def add_implicit_resolver(tag, regexp, first=None, + Loader=None, Dumper=Dumper): + """ + Add an implicit scalar detector. + If an implicit scalar value matches the given regexp, + the corresponding tag is assigned to the scalar. + first is a sequence of possible initial characters or None. + """ + if Loader is None: + loader.Loader.add_implicit_resolver(tag, regexp, first) + loader.FullLoader.add_implicit_resolver(tag, regexp, first) + loader.UnsafeLoader.add_implicit_resolver(tag, regexp, first) + else: + Loader.add_implicit_resolver(tag, regexp, first) + Dumper.add_implicit_resolver(tag, regexp, first) + +def add_path_resolver(tag, path, kind=None, Loader=None, Dumper=Dumper): + """ + Add a path based resolver for the given tag. + A path is a list of keys that forms a path + to a node in the representation tree. + Keys can be string values, integers, or None. + """ + if Loader is None: + loader.Loader.add_path_resolver(tag, path, kind) + loader.FullLoader.add_path_resolver(tag, path, kind) + loader.UnsafeLoader.add_path_resolver(tag, path, kind) + else: + Loader.add_path_resolver(tag, path, kind) + Dumper.add_path_resolver(tag, path, kind) + +def add_constructor(tag, constructor, Loader=None): + """ + Add a constructor for the given tag. + Constructor is a function that accepts a Loader instance + and a node object and produces the corresponding Python object. + """ + if Loader is None: + loader.Loader.add_constructor(tag, constructor) + loader.FullLoader.add_constructor(tag, constructor) + loader.UnsafeLoader.add_constructor(tag, constructor) + else: + Loader.add_constructor(tag, constructor) + +def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): + """ + Add a multi-constructor for the given tag prefix. + Multi-constructor is called for a node if its tag starts with tag_prefix. + Multi-constructor accepts a Loader instance, a tag suffix, + and a node object and produces the corresponding Python object. + """ + if Loader is None: + loader.Loader.add_multi_constructor(tag_prefix, multi_constructor) + loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor) + loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor) + else: + Loader.add_multi_constructor(tag_prefix, multi_constructor) + +def add_representer(data_type, representer, Dumper=Dumper): + """ + Add a representer for the given type. + Representer is a function accepting a Dumper instance + and an instance of the given data type + and producing the corresponding representation node. + """ + Dumper.add_representer(data_type, representer) + +def add_multi_representer(data_type, multi_representer, Dumper=Dumper): + """ + Add a representer for the given type. + Multi-representer is a function accepting a Dumper instance + and an instance of the given data type or subtype + and producing the corresponding representation node. + """ + Dumper.add_multi_representer(data_type, multi_representer) + +class YAMLObjectMetaclass(type): + """ + The metaclass for YAMLObject. + """ + def __init__(cls, name, bases, kwds): + super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds) + if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None: + if isinstance(cls.yaml_loader, list): + for loader in cls.yaml_loader: + loader.add_constructor(cls.yaml_tag, cls.from_yaml) + else: + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + + cls.yaml_dumper.add_representer(cls, cls.to_yaml) + +class YAMLObject(object): + """ + An object that can dump itself to a YAML stream + and load itself from a YAML stream. + """ + + __metaclass__ = YAMLObjectMetaclass + __slots__ = () # no direct instantiation, so allow immutable subclasses + + yaml_loader = [Loader, FullLoader, UnsafeLoader] + yaml_dumper = Dumper + + yaml_tag = None + yaml_flow_style = None + + def from_yaml(cls, loader, node): + """ + Convert a representation node to a Python object. + """ + return loader.construct_yaml_object(node, cls) + from_yaml = classmethod(from_yaml) + + def to_yaml(cls, dumper, data): + """ + Convert a Python object to a representation node. + """ + return dumper.represent_yaml_object(cls.yaml_tag, data, cls, + flow_style=cls.yaml_flow_style) + to_yaml = classmethod(to_yaml) + diff --git a/pipenv/patched/yaml2/composer.py b/pipenv/patched/yaml2/composer.py new file mode 100644 index 00000000..df85ef65 --- /dev/null +++ b/pipenv/patched/yaml2/composer.py @@ -0,0 +1,139 @@ + +__all__ = ['Composer', 'ComposerError'] + +from error import MarkedYAMLError +from events import * +from nodes import * + +class ComposerError(MarkedYAMLError): + pass + +class Composer(object): + + def __init__(self): + self.anchors = {} + + def check_node(self): + # Drop the STREAM-START event. + if self.check_event(StreamStartEvent): + self.get_event() + + # If there are more documents available? + return not self.check_event(StreamEndEvent) + + def get_node(self): + # Get the root node of the next document. + if not self.check_event(StreamEndEvent): + return self.compose_document() + + def get_single_node(self): + # Drop the STREAM-START event. + self.get_event() + + # Compose a document if the stream is not empty. + document = None + if not self.check_event(StreamEndEvent): + document = self.compose_document() + + # Ensure that the stream contains no more documents. + if not self.check_event(StreamEndEvent): + event = self.get_event() + raise ComposerError("expected a single document in the stream", + document.start_mark, "but found another document", + event.start_mark) + + # Drop the STREAM-END event. + self.get_event() + + return document + + def compose_document(self): + # Drop the DOCUMENT-START event. + self.get_event() + + # Compose the root node. + node = self.compose_node(None, None) + + # Drop the DOCUMENT-END event. + self.get_event() + + self.anchors = {} + return node + + def compose_node(self, parent, index): + if self.check_event(AliasEvent): + event = self.get_event() + anchor = event.anchor + if anchor not in self.anchors: + raise ComposerError(None, None, "found undefined alias %r" + % anchor.encode('utf-8'), event.start_mark) + return self.anchors[anchor] + event = self.peek_event() + anchor = event.anchor + if anchor is not None: + if anchor in self.anchors: + raise ComposerError("found duplicate anchor %r; first occurrence" + % anchor.encode('utf-8'), self.anchors[anchor].start_mark, + "second occurrence", event.start_mark) + self.descend_resolver(parent, index) + if self.check_event(ScalarEvent): + node = self.compose_scalar_node(anchor) + elif self.check_event(SequenceStartEvent): + node = self.compose_sequence_node(anchor) + elif self.check_event(MappingStartEvent): + node = self.compose_mapping_node(anchor) + self.ascend_resolver() + return node + + def compose_scalar_node(self, anchor): + event = self.get_event() + tag = event.tag + if tag is None or tag == u'!': + tag = self.resolve(ScalarNode, event.value, event.implicit) + node = ScalarNode(tag, event.value, + event.start_mark, event.end_mark, style=event.style) + if anchor is not None: + self.anchors[anchor] = node + return node + + def compose_sequence_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == u'!': + tag = self.resolve(SequenceNode, None, start_event.implicit) + node = SequenceNode(tag, [], + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + index = 0 + while not self.check_event(SequenceEndEvent): + node.value.append(self.compose_node(node, index)) + index += 1 + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + + def compose_mapping_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == u'!': + tag = self.resolve(MappingNode, None, start_event.implicit) + node = MappingNode(tag, [], + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + while not self.check_event(MappingEndEvent): + #key_event = self.peek_event() + item_key = self.compose_node(node, None) + #if item_key in node.value: + # raise ComposerError("while composing a mapping", start_event.start_mark, + # "found duplicate key", key_event.start_mark) + item_value = self.compose_node(node, item_key) + #node.value[item_key] = item_value + node.value.append((item_key, item_value)) + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + diff --git a/pipenv/patched/yaml2/constructor.py b/pipenv/patched/yaml2/constructor.py new file mode 100644 index 00000000..794681cb --- /dev/null +++ b/pipenv/patched/yaml2/constructor.py @@ -0,0 +1,760 @@ + +__all__ = [ + 'BaseConstructor', + 'SafeConstructor', + 'FullConstructor', + 'UnsafeConstructor', + 'Constructor', + 'ConstructorError' +] + +from error import * +from nodes import * + +import datetime + +import binascii, re, sys, types + +class ConstructorError(MarkedYAMLError): + pass + + +class timezone(datetime.tzinfo): + def __init__(self, offset): + self._offset = offset + seconds = abs(offset).total_seconds() + self._name = 'UTC%s%02d:%02d' % ( + '-' if offset.days < 0 else '+', + seconds // 3600, + seconds % 3600 // 60 + ) + + def tzname(self, dt=None): + return self._name + + def utcoffset(self, dt=None): + return self._offset + + def dst(self, dt=None): + return datetime.timedelta(0) + + __repr__ = __str__ = tzname + + +class BaseConstructor(object): + + yaml_constructors = {} + yaml_multi_constructors = {} + + def __init__(self): + self.constructed_objects = {} + self.recursive_objects = {} + self.state_generators = [] + self.deep_construct = False + + def check_data(self): + # If there are more documents available? + return self.check_node() + + def check_state_key(self, key): + """Block special attributes/methods from being set in a newly created + object, to prevent user-controlled methods from being called during + deserialization""" + if self.get_state_keys_blacklist_regexp().match(key): + raise ConstructorError(None, None, + "blacklisted key '%s' in instance state found" % (key,), None) + + def get_data(self): + # Construct and return the next document. + if self.check_node(): + return self.construct_document(self.get_node()) + + def get_single_data(self): + # Ensure that the stream contains a single document and construct it. + node = self.get_single_node() + if node is not None: + return self.construct_document(node) + return None + + def construct_document(self, node): + data = self.construct_object(node) + while self.state_generators: + state_generators = self.state_generators + self.state_generators = [] + for generator in state_generators: + for dummy in generator: + pass + self.constructed_objects = {} + self.recursive_objects = {} + self.deep_construct = False + return data + + def construct_object(self, node, deep=False): + if node in self.constructed_objects: + return self.constructed_objects[node] + if deep: + old_deep = self.deep_construct + self.deep_construct = True + if node in self.recursive_objects: + raise ConstructorError(None, None, + "found unconstructable recursive node", node.start_mark) + self.recursive_objects[node] = None + constructor = None + tag_suffix = None + if node.tag in self.yaml_constructors: + constructor = self.yaml_constructors[node.tag] + else: + for tag_prefix in self.yaml_multi_constructors: + if tag_prefix is not None and node.tag.startswith(tag_prefix): + tag_suffix = node.tag[len(tag_prefix):] + constructor = self.yaml_multi_constructors[tag_prefix] + break + else: + if None in self.yaml_multi_constructors: + tag_suffix = node.tag + constructor = self.yaml_multi_constructors[None] + elif None in self.yaml_constructors: + constructor = self.yaml_constructors[None] + elif isinstance(node, ScalarNode): + constructor = self.__class__.construct_scalar + elif isinstance(node, SequenceNode): + constructor = self.__class__.construct_sequence + elif isinstance(node, MappingNode): + constructor = self.__class__.construct_mapping + if tag_suffix is None: + data = constructor(self, node) + else: + data = constructor(self, tag_suffix, node) + if isinstance(data, types.GeneratorType): + generator = data + data = generator.next() + if self.deep_construct: + for dummy in generator: + pass + else: + self.state_generators.append(generator) + self.constructed_objects[node] = data + del self.recursive_objects[node] + if deep: + self.deep_construct = old_deep + return data + + def construct_scalar(self, node): + if not isinstance(node, ScalarNode): + raise ConstructorError(None, None, + "expected a scalar node, but found %s" % node.id, + node.start_mark) + return node.value + + def construct_sequence(self, node, deep=False): + if not isinstance(node, SequenceNode): + raise ConstructorError(None, None, + "expected a sequence node, but found %s" % node.id, + node.start_mark) + return [self.construct_object(child, deep=deep) + for child in node.value] + + def construct_mapping(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + mapping = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + try: + hash(key) + except TypeError, exc: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unacceptable key (%s)" % exc, key_node.start_mark) + value = self.construct_object(value_node, deep=deep) + mapping[key] = value + return mapping + + def construct_pairs(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + pairs = [] + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + value = self.construct_object(value_node, deep=deep) + pairs.append((key, value)) + return pairs + + def add_constructor(cls, tag, constructor): + if not 'yaml_constructors' in cls.__dict__: + cls.yaml_constructors = cls.yaml_constructors.copy() + cls.yaml_constructors[tag] = constructor + add_constructor = classmethod(add_constructor) + + def add_multi_constructor(cls, tag_prefix, multi_constructor): + if not 'yaml_multi_constructors' in cls.__dict__: + cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() + cls.yaml_multi_constructors[tag_prefix] = multi_constructor + add_multi_constructor = classmethod(add_multi_constructor) + +class SafeConstructor(BaseConstructor): + + def construct_scalar(self, node): + if isinstance(node, MappingNode): + for key_node, value_node in node.value: + if key_node.tag == u'tag:yaml.org,2002:value': + return self.construct_scalar(value_node) + return BaseConstructor.construct_scalar(self, node) + + def flatten_mapping(self, node): + merge = [] + index = 0 + while index < len(node.value): + key_node, value_node = node.value[index] + if key_node.tag == u'tag:yaml.org,2002:merge': + del node.value[index] + if isinstance(value_node, MappingNode): + self.flatten_mapping(value_node) + merge.extend(value_node.value) + elif isinstance(value_node, SequenceNode): + submerge = [] + for subnode in value_node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing a mapping", + node.start_mark, + "expected a mapping for merging, but found %s" + % subnode.id, subnode.start_mark) + self.flatten_mapping(subnode) + submerge.append(subnode.value) + submerge.reverse() + for value in submerge: + merge.extend(value) + else: + raise ConstructorError("while constructing a mapping", node.start_mark, + "expected a mapping or list of mappings for merging, but found %s" + % value_node.id, value_node.start_mark) + elif key_node.tag == u'tag:yaml.org,2002:value': + key_node.tag = u'tag:yaml.org,2002:str' + index += 1 + else: + index += 1 + if merge: + node.value = merge + node.value + + def construct_mapping(self, node, deep=False): + if isinstance(node, MappingNode): + self.flatten_mapping(node) + return BaseConstructor.construct_mapping(self, node, deep=deep) + + def construct_yaml_null(self, node): + self.construct_scalar(node) + return None + + bool_values = { + u'yes': True, + u'no': False, + u'true': True, + u'false': False, + u'on': True, + u'off': False, + } + + def construct_yaml_bool(self, node): + value = self.construct_scalar(node) + return self.bool_values[value.lower()] + + def construct_yaml_int(self, node): + value = str(self.construct_scalar(node)) + value = value.replace('_', '') + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '0': + return 0 + elif value.startswith('0b'): + return sign*int(value[2:], 2) + elif value.startswith('0x'): + return sign*int(value[2:], 16) + elif value[0] == '0': + return sign*int(value, 8) + elif ':' in value: + digits = [int(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*int(value) + + inf_value = 1e300 + while inf_value != inf_value*inf_value: + inf_value *= inf_value + nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99). + + def construct_yaml_float(self, node): + value = str(self.construct_scalar(node)) + value = value.replace('_', '').lower() + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '.inf': + return sign*self.inf_value + elif value == '.nan': + return self.nan_value + elif ':' in value: + digits = [float(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0.0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*float(value) + + def construct_yaml_binary(self, node): + value = self.construct_scalar(node) + try: + return str(value).decode('base64') + except (binascii.Error, UnicodeEncodeError), exc: + raise ConstructorError(None, None, + "failed to decode base64 data: %s" % exc, node.start_mark) + + timestamp_regexp = re.compile( + ur'''^(?P[0-9][0-9][0-9][0-9]) + -(?P[0-9][0-9]?) + -(?P[0-9][0-9]?) + (?:(?:[Tt]|[ \t]+) + (?P[0-9][0-9]?) + :(?P[0-9][0-9]) + :(?P[0-9][0-9]) + (?:\.(?P[0-9]*))? + (?:[ \t]*(?PZ|(?P[-+])(?P[0-9][0-9]?) + (?::(?P[0-9][0-9]))?))?)?$''', re.X) + + def construct_yaml_timestamp(self, node): + value = self.construct_scalar(node) + match = self.timestamp_regexp.match(node.value) + values = match.groupdict() + year = int(values['year']) + month = int(values['month']) + day = int(values['day']) + if not values['hour']: + return datetime.date(year, month, day) + hour = int(values['hour']) + minute = int(values['minute']) + second = int(values['second']) + fraction = 0 + tzinfo = None + if values['fraction']: + fraction = values['fraction'][:6] + while len(fraction) < 6: + fraction += '0' + fraction = int(fraction) + if values['tz_sign']: + tz_hour = int(values['tz_hour']) + tz_minute = int(values['tz_minute'] or 0) + delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute) + if values['tz_sign'] == '-': + delta = -delta + tzinfo = timezone(delta) + elif values['tz']: + tzinfo = timezone(datetime.timedelta(0)) + return datetime.datetime(year, month, day, hour, minute, second, fraction, + tzinfo=tzinfo) + + def construct_yaml_omap(self, node): + # Note: we do not check for duplicate keys, because it's too + # CPU-expensive. + omap = [] + yield omap + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node, value_node = subnode.value[0] + key = self.construct_object(key_node) + value = self.construct_object(value_node) + omap.append((key, value)) + + def construct_yaml_pairs(self, node): + # Note: the same code as `construct_yaml_omap`. + pairs = [] + yield pairs + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node, value_node = subnode.value[0] + key = self.construct_object(key_node) + value = self.construct_object(value_node) + pairs.append((key, value)) + + def construct_yaml_set(self, node): + data = set() + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_str(self, node): + value = self.construct_scalar(node) + try: + return value.encode('ascii') + except UnicodeEncodeError: + return value + + def construct_yaml_seq(self, node): + data = [] + yield data + data.extend(self.construct_sequence(node)) + + def construct_yaml_map(self, node): + data = {} + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_object(self, node, cls): + data = cls.__new__(cls) + yield data + if hasattr(data, '__setstate__'): + state = self.construct_mapping(node, deep=True) + data.__setstate__(state) + else: + state = self.construct_mapping(node) + data.__dict__.update(state) + + def construct_undefined(self, node): + raise ConstructorError(None, None, + "could not determine a constructor for the tag %r" % node.tag.encode('utf-8'), + node.start_mark) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:null', + SafeConstructor.construct_yaml_null) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:bool', + SafeConstructor.construct_yaml_bool) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:int', + SafeConstructor.construct_yaml_int) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:float', + SafeConstructor.construct_yaml_float) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:binary', + SafeConstructor.construct_yaml_binary) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:timestamp', + SafeConstructor.construct_yaml_timestamp) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:omap', + SafeConstructor.construct_yaml_omap) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:pairs', + SafeConstructor.construct_yaml_pairs) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:set', + SafeConstructor.construct_yaml_set) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:str', + SafeConstructor.construct_yaml_str) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:seq', + SafeConstructor.construct_yaml_seq) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:map', + SafeConstructor.construct_yaml_map) + +SafeConstructor.add_constructor(None, + SafeConstructor.construct_undefined) + +class FullConstructor(SafeConstructor): + # 'extend' is blacklisted because it is used by + # construct_python_object_apply to add `listitems` to a newly generate + # python instance + def get_state_keys_blacklist(self): + return ['^extend$', '^__.*__$'] + + def get_state_keys_blacklist_regexp(self): + if not hasattr(self, 'state_keys_blacklist_regexp'): + self.state_keys_blacklist_regexp = re.compile('(' + '|'.join(self.get_state_keys_blacklist()) + ')') + return self.state_keys_blacklist_regexp + + def construct_python_str(self, node): + return self.construct_scalar(node).encode('utf-8') + + def construct_python_unicode(self, node): + return self.construct_scalar(node) + + def construct_python_long(self, node): + return long(self.construct_yaml_int(node)) + + def construct_python_complex(self, node): + return complex(self.construct_scalar(node)) + + def construct_python_tuple(self, node): + return tuple(self.construct_sequence(node)) + + def find_python_module(self, name, mark, unsafe=False): + if not name: + raise ConstructorError("while constructing a Python module", mark, + "expected non-empty name appended to the tag", mark) + if unsafe: + try: + __import__(name) + except ImportError, exc: + raise ConstructorError("while constructing a Python module", mark, + "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark) + if name not in sys.modules: + raise ConstructorError("while constructing a Python module", mark, + "module %r is not imported" % name.encode('utf-8'), mark) + return sys.modules[name] + + def find_python_name(self, name, mark, unsafe=False): + if not name: + raise ConstructorError("while constructing a Python object", mark, + "expected non-empty name appended to the tag", mark) + if u'.' in name: + module_name, object_name = name.rsplit('.', 1) + else: + module_name = '__builtin__' + object_name = name + if unsafe: + try: + __import__(module_name) + except ImportError, exc: + raise ConstructorError("while constructing a Python object", mark, + "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark) + if module_name not in sys.modules: + raise ConstructorError("while constructing a Python object", mark, + "module %r is not imported" % module_name.encode('utf-8'), mark) + module = sys.modules[module_name] + if not hasattr(module, object_name): + raise ConstructorError("while constructing a Python object", mark, + "cannot find %r in the module %r" % (object_name.encode('utf-8'), + module.__name__), mark) + return getattr(module, object_name) + + def construct_python_name(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python name", node.start_mark, + "expected the empty value, but found %r" % value.encode('utf-8'), + node.start_mark) + return self.find_python_name(suffix, node.start_mark) + + def construct_python_module(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python module", node.start_mark, + "expected the empty value, but found %r" % value.encode('utf-8'), + node.start_mark) + return self.find_python_module(suffix, node.start_mark) + + class classobj: pass + + def make_python_instance(self, suffix, node, + args=None, kwds=None, newobj=False, unsafe=False): + if not args: + args = [] + if not kwds: + kwds = {} + cls = self.find_python_name(suffix, node.start_mark) + if not (unsafe or isinstance(cls, type) or isinstance(cls, type(self.classobj))): + raise ConstructorError("while constructing a Python instance", node.start_mark, + "expected a class, but found %r" % type(cls), + node.start_mark) + if newobj and isinstance(cls, type(self.classobj)) \ + and not args and not kwds: + instance = self.classobj() + instance.__class__ = cls + return instance + elif newobj and isinstance(cls, type): + return cls.__new__(cls, *args, **kwds) + else: + return cls(*args, **kwds) + + def set_python_instance_state(self, instance, state, unsafe=False): + if hasattr(instance, '__setstate__'): + instance.__setstate__(state) + else: + slotstate = {} + if isinstance(state, tuple) and len(state) == 2: + state, slotstate = state + if hasattr(instance, '__dict__'): + if not unsafe and state: + for key in state.keys(): + self.check_state_key(key) + instance.__dict__.update(state) + elif state: + slotstate.update(state) + for key, value in slotstate.items(): + if not unsafe: + self.check_state_key(key) + setattr(instance, key, value) + + def construct_python_object(self, suffix, node): + # Format: + # !!python/object:module.name { ... state ... } + instance = self.make_python_instance(suffix, node, newobj=True) + yield instance + deep = hasattr(instance, '__setstate__') + state = self.construct_mapping(node, deep=deep) + self.set_python_instance_state(instance, state) + + def construct_python_object_apply(self, suffix, node, newobj=False): + # Format: + # !!python/object/apply # (or !!python/object/new) + # args: [ ... arguments ... ] + # kwds: { ... keywords ... } + # state: ... state ... + # listitems: [ ... listitems ... ] + # dictitems: { ... dictitems ... } + # or short format: + # !!python/object/apply [ ... arguments ... ] + # The difference between !!python/object/apply and !!python/object/new + # is how an object is created, check make_python_instance for details. + if isinstance(node, SequenceNode): + args = self.construct_sequence(node, deep=True) + kwds = {} + state = {} + listitems = [] + dictitems = {} + else: + value = self.construct_mapping(node, deep=True) + args = value.get('args', []) + kwds = value.get('kwds', {}) + state = value.get('state', {}) + listitems = value.get('listitems', []) + dictitems = value.get('dictitems', {}) + instance = self.make_python_instance(suffix, node, args, kwds, newobj) + if state: + self.set_python_instance_state(instance, state) + if listitems: + instance.extend(listitems) + if dictitems: + for key in dictitems: + instance[key] = dictitems[key] + return instance + + def construct_python_object_new(self, suffix, node): + return self.construct_python_object_apply(suffix, node, newobj=True) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/none', + FullConstructor.construct_yaml_null) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/bool', + FullConstructor.construct_yaml_bool) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/str', + FullConstructor.construct_python_str) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/unicode', + FullConstructor.construct_python_unicode) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/int', + FullConstructor.construct_yaml_int) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/long', + FullConstructor.construct_python_long) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/float', + FullConstructor.construct_yaml_float) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/complex', + FullConstructor.construct_python_complex) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/list', + FullConstructor.construct_yaml_seq) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/tuple', + FullConstructor.construct_python_tuple) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/dict', + FullConstructor.construct_yaml_map) + +FullConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/name:', + FullConstructor.construct_python_name) + +FullConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/module:', + FullConstructor.construct_python_module) + +FullConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object:', + FullConstructor.construct_python_object) + +FullConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object/new:', + FullConstructor.construct_python_object_new) + +class UnsafeConstructor(FullConstructor): + + def find_python_module(self, name, mark): + return super(UnsafeConstructor, self).find_python_module(name, mark, unsafe=True) + + def find_python_name(self, name, mark): + return super(UnsafeConstructor, self).find_python_name(name, mark, unsafe=True) + + def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False): + return super(UnsafeConstructor, self).make_python_instance( + suffix, node, args, kwds, newobj, unsafe=True) + + def set_python_instance_state(self, instance, state): + return super(UnsafeConstructor, self).set_python_instance_state( + instance, state, unsafe=True) + +UnsafeConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object/apply:', + UnsafeConstructor.construct_python_object_apply) + +# Constructor is same as UnsafeConstructor. Need to leave this in place in case +# people have extended it directly. +class Constructor(UnsafeConstructor): + pass diff --git a/pipenv/patched/yaml2/cyaml.py b/pipenv/patched/yaml2/cyaml.py new file mode 100644 index 00000000..ebb89593 --- /dev/null +++ b/pipenv/patched/yaml2/cyaml.py @@ -0,0 +1,101 @@ + +__all__ = [ + 'CBaseLoader', 'CSafeLoader', 'CFullLoader', 'CUnsafeLoader', 'CLoader', + 'CBaseDumper', 'CSafeDumper', 'CDumper' +] + +from _yaml import CParser, CEmitter + +from constructor import * + +from serializer import * +from representer import * + +from resolver import * + +class CBaseLoader(CParser, BaseConstructor, BaseResolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + BaseConstructor.__init__(self) + BaseResolver.__init__(self) + +class CSafeLoader(CParser, SafeConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + SafeConstructor.__init__(self) + Resolver.__init__(self) + +class CFullLoader(CParser, FullConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + FullConstructor.__init__(self) + Resolver.__init__(self) + +class CUnsafeLoader(CParser, UnsafeConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + UnsafeConstructor.__init__(self) + Resolver.__init__(self) + +class CLoader(CParser, Constructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + Constructor.__init__(self) + Resolver.__init__(self) + +class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class CSafeDumper(CEmitter, SafeRepresenter, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + SafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class CDumper(CEmitter, Serializer, Representer, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + diff --git a/pipenv/patched/yaml2/dumper.py b/pipenv/patched/yaml2/dumper.py new file mode 100644 index 00000000..f9cd49fd --- /dev/null +++ b/pipenv/patched/yaml2/dumper.py @@ -0,0 +1,62 @@ + +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper'] + +from emitter import * +from serializer import * +from representer import * +from resolver import * + +class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + SafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class Dumper(Emitter, Serializer, Representer, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + diff --git a/pipenv/patched/yaml2/emitter.py b/pipenv/patched/yaml2/emitter.py new file mode 100644 index 00000000..23c25ca8 --- /dev/null +++ b/pipenv/patched/yaml2/emitter.py @@ -0,0 +1,1144 @@ + +# Emitter expects events obeying the following grammar: +# stream ::= STREAM-START document* STREAM-END +# document ::= DOCUMENT-START node DOCUMENT-END +# node ::= SCALAR | sequence | mapping +# sequence ::= SEQUENCE-START node* SEQUENCE-END +# mapping ::= MAPPING-START (node node)* MAPPING-END + +__all__ = ['Emitter', 'EmitterError'] + +import sys + +from error import YAMLError +from events import * + +has_ucs4 = sys.maxunicode > 0xffff + +class EmitterError(YAMLError): + pass + +class ScalarAnalysis(object): + def __init__(self, scalar, empty, multiline, + allow_flow_plain, allow_block_plain, + allow_single_quoted, allow_double_quoted, + allow_block): + self.scalar = scalar + self.empty = empty + self.multiline = multiline + self.allow_flow_plain = allow_flow_plain + self.allow_block_plain = allow_block_plain + self.allow_single_quoted = allow_single_quoted + self.allow_double_quoted = allow_double_quoted + self.allow_block = allow_block + +class Emitter(object): + + DEFAULT_TAG_PREFIXES = { + u'!' : u'!', + u'tag:yaml.org,2002:' : u'!!', + } + + def __init__(self, stream, canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + + # The stream should have the methods `write` and possibly `flush`. + self.stream = stream + + # Encoding can be overridden by STREAM-START. + self.encoding = None + + # Emitter is a state machine with a stack of states to handle nested + # structures. + self.states = [] + self.state = self.expect_stream_start + + # Current event and the event queue. + self.events = [] + self.event = None + + # The current indentation level and the stack of previous indents. + self.indents = [] + self.indent = None + + # Flow level. + self.flow_level = 0 + + # Contexts. + self.root_context = False + self.sequence_context = False + self.mapping_context = False + self.simple_key_context = False + + # Characteristics of the last emitted character: + # - current position. + # - is it a whitespace? + # - is it an indention character + # (indentation space, '-', '?', or ':')? + self.line = 0 + self.column = 0 + self.whitespace = True + self.indention = True + + # Whether the document requires an explicit document indicator + self.open_ended = False + + # Formatting details. + self.canonical = canonical + self.allow_unicode = allow_unicode + self.best_indent = 2 + if indent and 1 < indent < 10: + self.best_indent = indent + self.best_width = 80 + if width and width > self.best_indent*2: + self.best_width = width + self.best_line_break = u'\n' + if line_break in [u'\r', u'\n', u'\r\n']: + self.best_line_break = line_break + + # Tag prefixes. + self.tag_prefixes = None + + # Prepared anchor and tag. + self.prepared_anchor = None + self.prepared_tag = None + + # Scalar analysis and style. + self.analysis = None + self.style = None + + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + + def emit(self, event): + self.events.append(event) + while not self.need_more_events(): + self.event = self.events.pop(0) + self.state() + self.event = None + + # In some cases, we wait for a few next events before emitting. + + def need_more_events(self): + if not self.events: + return True + event = self.events[0] + if isinstance(event, DocumentStartEvent): + return self.need_events(1) + elif isinstance(event, SequenceStartEvent): + return self.need_events(2) + elif isinstance(event, MappingStartEvent): + return self.need_events(3) + else: + return False + + def need_events(self, count): + level = 0 + for event in self.events[1:]: + if isinstance(event, (DocumentStartEvent, CollectionStartEvent)): + level += 1 + elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)): + level -= 1 + elif isinstance(event, StreamEndEvent): + level = -1 + if level < 0: + return False + return (len(self.events) < count+1) + + def increase_indent(self, flow=False, indentless=False): + self.indents.append(self.indent) + if self.indent is None: + if flow: + self.indent = self.best_indent + else: + self.indent = 0 + elif not indentless: + self.indent += self.best_indent + + # States. + + # Stream handlers. + + def expect_stream_start(self): + if isinstance(self.event, StreamStartEvent): + if self.event.encoding and not getattr(self.stream, 'encoding', None): + self.encoding = self.event.encoding + self.write_stream_start() + self.state = self.expect_first_document_start + else: + raise EmitterError("expected StreamStartEvent, but got %s" + % self.event) + + def expect_nothing(self): + raise EmitterError("expected nothing, but got %s" % self.event) + + # Document handlers. + + def expect_first_document_start(self): + return self.expect_document_start(first=True) + + def expect_document_start(self, first=False): + if isinstance(self.event, DocumentStartEvent): + if (self.event.version or self.event.tags) and self.open_ended: + self.write_indicator(u'...', True) + self.write_indent() + if self.event.version: + version_text = self.prepare_version(self.event.version) + self.write_version_directive(version_text) + self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy() + if self.event.tags: + handles = self.event.tags.keys() + handles.sort() + for handle in handles: + prefix = self.event.tags[handle] + self.tag_prefixes[prefix] = handle + handle_text = self.prepare_tag_handle(handle) + prefix_text = self.prepare_tag_prefix(prefix) + self.write_tag_directive(handle_text, prefix_text) + implicit = (first and not self.event.explicit and not self.canonical + and not self.event.version and not self.event.tags + and not self.check_empty_document()) + if not implicit: + self.write_indent() + self.write_indicator(u'---', True) + if self.canonical: + self.write_indent() + self.state = self.expect_document_root + elif isinstance(self.event, StreamEndEvent): + if self.open_ended: + self.write_indicator(u'...', True) + self.write_indent() + self.write_stream_end() + self.state = self.expect_nothing + else: + raise EmitterError("expected DocumentStartEvent, but got %s" + % self.event) + + def expect_document_end(self): + if isinstance(self.event, DocumentEndEvent): + self.write_indent() + if self.event.explicit: + self.write_indicator(u'...', True) + self.write_indent() + self.flush_stream() + self.state = self.expect_document_start + else: + raise EmitterError("expected DocumentEndEvent, but got %s" + % self.event) + + def expect_document_root(self): + self.states.append(self.expect_document_end) + self.expect_node(root=True) + + # Node handlers. + + def expect_node(self, root=False, sequence=False, mapping=False, + simple_key=False): + self.root_context = root + self.sequence_context = sequence + self.mapping_context = mapping + self.simple_key_context = simple_key + if isinstance(self.event, AliasEvent): + self.expect_alias() + elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)): + self.process_anchor(u'&') + self.process_tag() + if isinstance(self.event, ScalarEvent): + self.expect_scalar() + elif isinstance(self.event, SequenceStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_sequence(): + self.expect_flow_sequence() + else: + self.expect_block_sequence() + elif isinstance(self.event, MappingStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_mapping(): + self.expect_flow_mapping() + else: + self.expect_block_mapping() + else: + raise EmitterError("expected NodeEvent, but got %s" % self.event) + + def expect_alias(self): + if self.event.anchor is None: + raise EmitterError("anchor is not specified for alias") + self.process_anchor(u'*') + self.state = self.states.pop() + + def expect_scalar(self): + self.increase_indent(flow=True) + self.process_scalar() + self.indent = self.indents.pop() + self.state = self.states.pop() + + # Flow sequence handlers. + + def expect_flow_sequence(self): + self.write_indicator(u'[', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_sequence_item + + def expect_first_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator(u']', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + def expect_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(u',', False) + self.write_indent() + self.write_indicator(u']', False) + self.state = self.states.pop() + else: + self.write_indicator(u',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + # Flow mapping handlers. + + def expect_flow_mapping(self): + self.write_indicator(u'{', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_mapping_key + + def expect_first_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator(u'}', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator(u'?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(u',', False) + self.write_indent() + self.write_indicator(u'}', False) + self.state = self.states.pop() + else: + self.write_indicator(u',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator(u'?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_simple_value(self): + self.write_indicator(u':', False) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + def expect_flow_mapping_value(self): + if self.canonical or self.column > self.best_width: + self.write_indent() + self.write_indicator(u':', True) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + # Block sequence handlers. + + def expect_block_sequence(self): + indentless = (self.mapping_context and not self.indention) + self.increase_indent(flow=False, indentless=indentless) + self.state = self.expect_first_block_sequence_item + + def expect_first_block_sequence_item(self): + return self.expect_block_sequence_item(first=True) + + def expect_block_sequence_item(self, first=False): + if not first and isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + self.write_indicator(u'-', True, indention=True) + self.states.append(self.expect_block_sequence_item) + self.expect_node(sequence=True) + + # Block mapping handlers. + + def expect_block_mapping(self): + self.increase_indent(flow=False) + self.state = self.expect_first_block_mapping_key + + def expect_first_block_mapping_key(self): + return self.expect_block_mapping_key(first=True) + + def expect_block_mapping_key(self, first=False): + if not first and isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + if self.check_simple_key(): + self.states.append(self.expect_block_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator(u'?', True, indention=True) + self.states.append(self.expect_block_mapping_value) + self.expect_node(mapping=True) + + def expect_block_mapping_simple_value(self): + self.write_indicator(u':', False) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + def expect_block_mapping_value(self): + self.write_indent() + self.write_indicator(u':', True, indention=True) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + # Checkers. + + def check_empty_sequence(self): + return (isinstance(self.event, SequenceStartEvent) and self.events + and isinstance(self.events[0], SequenceEndEvent)) + + def check_empty_mapping(self): + return (isinstance(self.event, MappingStartEvent) and self.events + and isinstance(self.events[0], MappingEndEvent)) + + def check_empty_document(self): + if not isinstance(self.event, DocumentStartEvent) or not self.events: + return False + event = self.events[0] + return (isinstance(event, ScalarEvent) and event.anchor is None + and event.tag is None and event.implicit and event.value == u'') + + def check_simple_key(self): + length = 0 + if isinstance(self.event, NodeEvent) and self.event.anchor is not None: + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + length += len(self.prepared_anchor) + if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \ + and self.event.tag is not None: + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(self.event.tag) + length += len(self.prepared_tag) + if isinstance(self.event, ScalarEvent): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + length += len(self.analysis.scalar) + return (length < 128 and (isinstance(self.event, AliasEvent) + or (isinstance(self.event, ScalarEvent) + and not self.analysis.empty and not self.analysis.multiline) + or self.check_empty_sequence() or self.check_empty_mapping())) + + # Anchor, Tag, and Scalar processors. + + def process_anchor(self, indicator): + if self.event.anchor is None: + self.prepared_anchor = None + return + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + if self.prepared_anchor: + self.write_indicator(indicator+self.prepared_anchor, True) + self.prepared_anchor = None + + def process_tag(self): + tag = self.event.tag + if isinstance(self.event, ScalarEvent): + if self.style is None: + self.style = self.choose_scalar_style() + if ((not self.canonical or tag is None) and + ((self.style == '' and self.event.implicit[0]) + or (self.style != '' and self.event.implicit[1]))): + self.prepared_tag = None + return + if self.event.implicit[0] and tag is None: + tag = u'!' + self.prepared_tag = None + else: + if (not self.canonical or tag is None) and self.event.implicit: + self.prepared_tag = None + return + if tag is None: + raise EmitterError("tag is not specified") + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(tag) + if self.prepared_tag: + self.write_indicator(self.prepared_tag, True) + self.prepared_tag = None + + def choose_scalar_style(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.event.style == '"' or self.canonical: + return '"' + if not self.event.style and self.event.implicit[0]: + if (not (self.simple_key_context and + (self.analysis.empty or self.analysis.multiline)) + and (self.flow_level and self.analysis.allow_flow_plain + or (not self.flow_level and self.analysis.allow_block_plain))): + return '' + if self.event.style and self.event.style in '|>': + if (not self.flow_level and not self.simple_key_context + and self.analysis.allow_block): + return self.event.style + if not self.event.style or self.event.style == '\'': + if (self.analysis.allow_single_quoted and + not (self.simple_key_context and self.analysis.multiline)): + return '\'' + return '"' + + def process_scalar(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.style is None: + self.style = self.choose_scalar_style() + split = (not self.simple_key_context) + #if self.analysis.multiline and split \ + # and (not self.style or self.style in '\'\"'): + # self.write_indent() + if self.style == '"': + self.write_double_quoted(self.analysis.scalar, split) + elif self.style == '\'': + self.write_single_quoted(self.analysis.scalar, split) + elif self.style == '>': + self.write_folded(self.analysis.scalar) + elif self.style == '|': + self.write_literal(self.analysis.scalar) + else: + self.write_plain(self.analysis.scalar, split) + self.analysis = None + self.style = None + + # Analyzers. + + def prepare_version(self, version): + major, minor = version + if major != 1: + raise EmitterError("unsupported YAML version: %d.%d" % (major, minor)) + return u'%d.%d' % (major, minor) + + def prepare_tag_handle(self, handle): + if not handle: + raise EmitterError("tag handle must not be empty") + if handle[0] != u'!' or handle[-1] != u'!': + raise EmitterError("tag handle must start and end with '!': %r" + % (handle.encode('utf-8'))) + for ch in handle[1:-1]: + if not (u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-_'): + raise EmitterError("invalid character %r in the tag handle: %r" + % (ch.encode('utf-8'), handle.encode('utf-8'))) + return handle + + def prepare_tag_prefix(self, prefix): + if not prefix: + raise EmitterError("tag prefix must not be empty") + chunks = [] + start = end = 0 + if prefix[0] == u'!': + end = 1 + while end < len(prefix): + ch = prefix[end] + if u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-;/?!:@&=+$,_.~*\'()[]': + end += 1 + else: + if start < end: + chunks.append(prefix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append(u'%%%02X' % ord(ch)) + if start < end: + chunks.append(prefix[start:end]) + return u''.join(chunks) + + def prepare_tag(self, tag): + if not tag: + raise EmitterError("tag must not be empty") + if tag == u'!': + return tag + handle = None + suffix = tag + prefixes = self.tag_prefixes.keys() + prefixes.sort() + for prefix in prefixes: + if tag.startswith(prefix) \ + and (prefix == u'!' or len(prefix) < len(tag)): + handle = self.tag_prefixes[prefix] + suffix = tag[len(prefix):] + chunks = [] + start = end = 0 + while end < len(suffix): + ch = suffix[end] + if u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-;/?:@&=+$,_.~*\'()[]' \ + or (ch == u'!' and handle != u'!'): + end += 1 + else: + if start < end: + chunks.append(suffix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append(u'%%%02X' % ord(ch)) + if start < end: + chunks.append(suffix[start:end]) + suffix_text = u''.join(chunks) + if handle: + return u'%s%s' % (handle, suffix_text) + else: + return u'!<%s>' % suffix_text + + def prepare_anchor(self, anchor): + if not anchor: + raise EmitterError("anchor must not be empty") + for ch in anchor: + if not (u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-_'): + raise EmitterError("invalid character %r in the anchor: %r" + % (ch.encode('utf-8'), anchor.encode('utf-8'))) + return anchor + + def analyze_scalar(self, scalar): + + # Empty scalar is a special case. + if not scalar: + return ScalarAnalysis(scalar=scalar, empty=True, multiline=False, + allow_flow_plain=False, allow_block_plain=True, + allow_single_quoted=True, allow_double_quoted=True, + allow_block=False) + + # Indicators and special characters. + block_indicators = False + flow_indicators = False + line_breaks = False + special_characters = False + + # Important whitespace combinations. + leading_space = False + leading_break = False + trailing_space = False + trailing_break = False + break_space = False + space_break = False + + # Check document indicators. + if scalar.startswith(u'---') or scalar.startswith(u'...'): + block_indicators = True + flow_indicators = True + + # First character or preceded by a whitespace. + preceded_by_whitespace = True + + # Last character or followed by a whitespace. + followed_by_whitespace = (len(scalar) == 1 or + scalar[1] in u'\0 \t\r\n\x85\u2028\u2029') + + # The previous character is a space. + previous_space = False + + # The previous character is a break. + previous_break = False + + index = 0 + while index < len(scalar): + ch = scalar[index] + + # Check for indicators. + if index == 0: + # Leading indicators are special characters. + if ch in u'#,[]{}&*!|>\'\"%@`': + flow_indicators = True + block_indicators = True + if ch in u'?:': + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == u'-' and followed_by_whitespace: + flow_indicators = True + block_indicators = True + else: + # Some indicators cannot appear within a scalar as well. + if ch in u',?[]{}': + flow_indicators = True + if ch == u':': + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == u'#' and preceded_by_whitespace: + flow_indicators = True + block_indicators = True + + # Check for line breaks, special, and unicode characters. + if ch in u'\n\x85\u2028\u2029': + line_breaks = True + if not (ch == u'\n' or u'\x20' <= ch <= u'\x7E'): + if (ch == u'\x85' or u'\xA0' <= ch <= u'\uD7FF' + or u'\uE000' <= ch <= u'\uFFFD' + or (u'\U00010000' <= ch < u'\U0010ffff')) and ch != u'\uFEFF': + unicode_characters = True + if not self.allow_unicode: + special_characters = True + else: + special_characters = True + + # Detect important whitespace combinations. + if ch == u' ': + if index == 0: + leading_space = True + if index == len(scalar)-1: + trailing_space = True + if previous_break: + break_space = True + previous_space = True + previous_break = False + elif ch in u'\n\x85\u2028\u2029': + if index == 0: + leading_break = True + if index == len(scalar)-1: + trailing_break = True + if previous_space: + space_break = True + previous_space = False + previous_break = True + else: + previous_space = False + previous_break = False + + # Prepare for the next character. + index += 1 + preceded_by_whitespace = (ch in u'\0 \t\r\n\x85\u2028\u2029') + followed_by_whitespace = (index+1 >= len(scalar) or + scalar[index+1] in u'\0 \t\r\n\x85\u2028\u2029') + + # Let's decide what styles are allowed. + allow_flow_plain = True + allow_block_plain = True + allow_single_quoted = True + allow_double_quoted = True + allow_block = True + + # Leading and trailing whitespaces are bad for plain scalars. + if (leading_space or leading_break + or trailing_space or trailing_break): + allow_flow_plain = allow_block_plain = False + + # We do not permit trailing spaces for block scalars. + if trailing_space: + allow_block = False + + # Spaces at the beginning of a new line are only acceptable for block + # scalars. + if break_space: + allow_flow_plain = allow_block_plain = allow_single_quoted = False + + # Spaces followed by breaks, as well as special character are only + # allowed for double quoted scalars. + if space_break or special_characters: + allow_flow_plain = allow_block_plain = \ + allow_single_quoted = allow_block = False + + # Although the plain scalar writer supports breaks, we never emit + # multiline plain scalars. + if line_breaks: + allow_flow_plain = allow_block_plain = False + + # Flow indicators are forbidden for flow plain scalars. + if flow_indicators: + allow_flow_plain = False + + # Block indicators are forbidden for block plain scalars. + if block_indicators: + allow_block_plain = False + + return ScalarAnalysis(scalar=scalar, + empty=False, multiline=line_breaks, + allow_flow_plain=allow_flow_plain, + allow_block_plain=allow_block_plain, + allow_single_quoted=allow_single_quoted, + allow_double_quoted=allow_double_quoted, + allow_block=allow_block) + + # Writers. + + def flush_stream(self): + if hasattr(self.stream, 'flush'): + self.stream.flush() + + def write_stream_start(self): + # Write BOM if needed. + if self.encoding and self.encoding.startswith('utf-16'): + self.stream.write(u'\uFEFF'.encode(self.encoding)) + + def write_stream_end(self): + self.flush_stream() + + def write_indicator(self, indicator, need_whitespace, + whitespace=False, indention=False): + if self.whitespace or not need_whitespace: + data = indicator + else: + data = u' '+indicator + self.whitespace = whitespace + self.indention = self.indention and indention + self.column += len(data) + self.open_ended = False + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_indent(self): + indent = self.indent or 0 + if not self.indention or self.column > indent \ + or (self.column == indent and not self.whitespace): + self.write_line_break() + if self.column < indent: + self.whitespace = True + data = u' '*(indent-self.column) + self.column = indent + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_line_break(self, data=None): + if data is None: + data = self.best_line_break + self.whitespace = True + self.indention = True + self.line += 1 + self.column = 0 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_version_directive(self, version_text): + data = u'%%YAML %s' % version_text + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + def write_tag_directive(self, handle_text, prefix_text): + data = u'%%TAG %s %s' % (handle_text, prefix_text) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + # Scalar streams. + + def write_single_quoted(self, text, split=True): + self.write_indicator(u'\'', True) + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch is None or ch != u' ': + if start+1 == end and self.column > self.best_width and split \ + and start != 0 and end != len(text): + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch is None or ch not in u'\n\x85\u2028\u2029': + if text[start] == u'\n': + self.write_line_break() + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + start = end + else: + if ch is None or ch in u' \n\x85\u2028\u2029' or ch == u'\'': + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch == u'\'': + data = u'\'\'' + self.column += 2 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + 1 + if ch is not None: + spaces = (ch == u' ') + breaks = (ch in u'\n\x85\u2028\u2029') + end += 1 + self.write_indicator(u'\'', False) + + ESCAPE_REPLACEMENTS = { + u'\0': u'0', + u'\x07': u'a', + u'\x08': u'b', + u'\x09': u't', + u'\x0A': u'n', + u'\x0B': u'v', + u'\x0C': u'f', + u'\x0D': u'r', + u'\x1B': u'e', + u'\"': u'\"', + u'\\': u'\\', + u'\x85': u'N', + u'\xA0': u'_', + u'\u2028': u'L', + u'\u2029': u'P', + } + + def write_double_quoted(self, text, split=True): + self.write_indicator(u'"', True) + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if ch is None or ch in u'"\\\x85\u2028\u2029\uFEFF' \ + or not (u'\x20' <= ch <= u'\x7E' + or (self.allow_unicode + and (u'\xA0' <= ch <= u'\uD7FF' + or u'\uE000' <= ch <= u'\uFFFD'))): + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + if ch in self.ESCAPE_REPLACEMENTS: + data = u'\\'+self.ESCAPE_REPLACEMENTS[ch] + elif ch <= u'\xFF': + data = u'\\x%02X' % ord(ch) + elif ch <= u'\uFFFF': + data = u'\\u%04X' % ord(ch) + else: + data = u'\\U%08X' % ord(ch) + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end+1 + if 0 < end < len(text)-1 and (ch == u' ' or start >= end) \ + and self.column+(end-start) > self.best_width and split: + data = text[start:end]+u'\\' + if start < end: + start = end + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_indent() + self.whitespace = False + self.indention = False + if text[start] == u' ': + data = u'\\' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + end += 1 + self.write_indicator(u'"', False) + + def determine_block_hints(self, text): + hints = u'' + if text: + if text[0] in u' \n\x85\u2028\u2029': + hints += unicode(self.best_indent) + if text[-1] not in u'\n\x85\u2028\u2029': + hints += u'-' + elif len(text) == 1 or text[-2] in u'\n\x85\u2028\u2029': + hints += u'+' + return hints + + def write_folded(self, text): + hints = self.determine_block_hints(text) + self.write_indicator(u'>'+hints, True) + if hints[-1:] == u'+': + self.open_ended = True + self.write_line_break() + leading_space = True + spaces = False + breaks = True + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in u'\n\x85\u2028\u2029': + if not leading_space and ch is not None and ch != u' ' \ + and text[start] == u'\n': + self.write_line_break() + leading_space = (ch == u' ') + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + elif spaces: + if ch != u' ': + if start+1 == end and self.column > self.best_width: + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + else: + if ch is None or ch in u' \n\x85\u2028\u2029': + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in u'\n\x85\u2028\u2029') + spaces = (ch == u' ') + end += 1 + + def write_literal(self, text): + hints = self.determine_block_hints(text) + self.write_indicator(u'|'+hints, True) + if hints[-1:] == u'+': + self.open_ended = True + self.write_line_break() + breaks = True + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in u'\n\x85\u2028\u2029': + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + else: + if ch is None or ch in u'\n\x85\u2028\u2029': + data = text[start:end] + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in u'\n\x85\u2028\u2029') + end += 1 + + def write_plain(self, text, split=True): + if self.root_context: + self.open_ended = True + if not text: + return + if not self.whitespace: + data = u' ' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.whitespace = False + self.indention = False + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch != u' ': + if start+1 == end and self.column > self.best_width and split: + self.write_indent() + self.whitespace = False + self.indention = False + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch not in u'\n\x85\u2028\u2029': + if text[start] == u'\n': + self.write_line_break() + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + self.whitespace = False + self.indention = False + start = end + else: + if ch is None or ch in u' \n\x85\u2028\u2029': + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + spaces = (ch == u' ') + breaks = (ch in u'\n\x85\u2028\u2029') + end += 1 diff --git a/pipenv/patched/yaml2/error.py b/pipenv/patched/yaml2/error.py new file mode 100644 index 00000000..577686db --- /dev/null +++ b/pipenv/patched/yaml2/error.py @@ -0,0 +1,75 @@ + +__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError'] + +class Mark(object): + + def __init__(self, name, index, line, column, buffer, pointer): + self.name = name + self.index = index + self.line = line + self.column = column + self.buffer = buffer + self.pointer = pointer + + def get_snippet(self, indent=4, max_length=75): + if self.buffer is None: + return None + head = '' + start = self.pointer + while start > 0 and self.buffer[start-1] not in u'\0\r\n\x85\u2028\u2029': + start -= 1 + if self.pointer-start > max_length/2-1: + head = ' ... ' + start += 5 + break + tail = '' + end = self.pointer + while end < len(self.buffer) and self.buffer[end] not in u'\0\r\n\x85\u2028\u2029': + end += 1 + if end-self.pointer > max_length/2-1: + tail = ' ... ' + end -= 5 + break + snippet = self.buffer[start:end].encode('utf-8') + return ' '*indent + head + snippet + tail + '\n' \ + + ' '*(indent+self.pointer-start+len(head)) + '^' + + def __str__(self): + snippet = self.get_snippet() + where = " in \"%s\", line %d, column %d" \ + % (self.name, self.line+1, self.column+1) + if snippet is not None: + where += ":\n"+snippet + return where + +class YAMLError(Exception): + pass + +class MarkedYAMLError(YAMLError): + + def __init__(self, context=None, context_mark=None, + problem=None, problem_mark=None, note=None): + self.context = context + self.context_mark = context_mark + self.problem = problem + self.problem_mark = problem_mark + self.note = note + + def __str__(self): + lines = [] + if self.context is not None: + lines.append(self.context) + if self.context_mark is not None \ + and (self.problem is None or self.problem_mark is None + or self.context_mark.name != self.problem_mark.name + or self.context_mark.line != self.problem_mark.line + or self.context_mark.column != self.problem_mark.column): + lines.append(str(self.context_mark)) + if self.problem is not None: + lines.append(self.problem) + if self.problem_mark is not None: + lines.append(str(self.problem_mark)) + if self.note is not None: + lines.append(self.note) + return '\n'.join(lines) + diff --git a/pipenv/patched/yaml2/events.py b/pipenv/patched/yaml2/events.py new file mode 100644 index 00000000..f79ad389 --- /dev/null +++ b/pipenv/patched/yaml2/events.py @@ -0,0 +1,86 @@ + +# Abstract classes. + +class Event(object): + def __init__(self, start_mark=None, end_mark=None): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in ['anchor', 'tag', 'implicit', 'value'] + if hasattr(self, key)] + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +class NodeEvent(Event): + def __init__(self, anchor, start_mark=None, end_mark=None): + self.anchor = anchor + self.start_mark = start_mark + self.end_mark = end_mark + +class CollectionStartEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None, + flow_style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class CollectionEndEvent(Event): + pass + +# Implementations. + +class StreamStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndEvent(Event): + pass + +class DocumentStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None, version=None, tags=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + self.version = version + self.tags = tags + +class DocumentEndEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + +class AliasEvent(NodeEvent): + pass + +class ScalarEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, value, + start_mark=None, end_mark=None, style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class SequenceStartEvent(CollectionStartEvent): + pass + +class SequenceEndEvent(CollectionEndEvent): + pass + +class MappingStartEvent(CollectionStartEvent): + pass + +class MappingEndEvent(CollectionEndEvent): + pass + diff --git a/pipenv/patched/yaml2/loader.py b/pipenv/patched/yaml2/loader.py new file mode 100644 index 00000000..4d773c3c --- /dev/null +++ b/pipenv/patched/yaml2/loader.py @@ -0,0 +1,63 @@ + +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader'] + +from reader import * +from scanner import * +from parser import * +from composer import * +from constructor import * +from resolver import * + +class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + BaseConstructor.__init__(self) + BaseResolver.__init__(self) + +class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + FullConstructor.__init__(self) + Resolver.__init__(self) + +class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + SafeConstructor.__init__(self) + Resolver.__init__(self) + +class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) + +# UnsafeLoader is the same as Loader (which is and was always unsafe on +# untrusted input). Use of either Loader or UnsafeLoader should be rare, since +# FullLoad should be able to load almost all YAML safely. Loader is left intact +# to ensure backwards compatibility. +class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) diff --git a/pipenv/patched/yaml2/nodes.py b/pipenv/patched/yaml2/nodes.py new file mode 100644 index 00000000..c4f070c4 --- /dev/null +++ b/pipenv/patched/yaml2/nodes.py @@ -0,0 +1,49 @@ + +class Node(object): + def __init__(self, tag, value, start_mark, end_mark): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + value = self.value + #if isinstance(value, list): + # if len(value) == 0: + # value = '' + # elif len(value) == 1: + # value = '<1 item>' + # else: + # value = '<%d items>' % len(value) + #else: + # if len(value) > 75: + # value = repr(value[:70]+u' ... ') + # else: + # value = repr(value) + value = repr(value) + return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value) + +class ScalarNode(Node): + id = 'scalar' + def __init__(self, tag, value, + start_mark=None, end_mark=None, style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class CollectionNode(Node): + def __init__(self, tag, value, + start_mark=None, end_mark=None, flow_style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class SequenceNode(CollectionNode): + id = 'sequence' + +class MappingNode(CollectionNode): + id = 'mapping' + diff --git a/pipenv/patched/yaml2/parser.py b/pipenv/patched/yaml2/parser.py new file mode 100644 index 00000000..f9e3057f --- /dev/null +++ b/pipenv/patched/yaml2/parser.py @@ -0,0 +1,589 @@ + +# The following YAML grammar is LL(1) and is parsed by a recursive descent +# parser. +# +# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +# implicit_document ::= block_node DOCUMENT-END* +# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +# block_node_or_indentless_sequence ::= +# ALIAS +# | properties (block_content | indentless_block_sequence)? +# | block_content +# | indentless_block_sequence +# block_node ::= ALIAS +# | properties block_content? +# | block_content +# flow_node ::= ALIAS +# | properties flow_content? +# | flow_content +# properties ::= TAG ANCHOR? | ANCHOR TAG? +# block_content ::= block_collection | flow_collection | SCALAR +# flow_content ::= flow_collection | SCALAR +# block_collection ::= block_sequence | block_mapping +# flow_collection ::= flow_sequence | flow_mapping +# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +# indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +# block_mapping ::= BLOCK-MAPPING_START +# ((KEY block_node_or_indentless_sequence?)? +# (VALUE block_node_or_indentless_sequence?)?)* +# BLOCK-END +# flow_sequence ::= FLOW-SEQUENCE-START +# (flow_sequence_entry FLOW-ENTRY)* +# flow_sequence_entry? +# FLOW-SEQUENCE-END +# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# flow_mapping ::= FLOW-MAPPING-START +# (flow_mapping_entry FLOW-ENTRY)* +# flow_mapping_entry? +# FLOW-MAPPING-END +# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# +# FIRST sets: +# +# stream: { STREAM-START } +# explicit_document: { DIRECTIVE DOCUMENT-START } +# implicit_document: FIRST(block_node) +# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_sequence: { BLOCK-SEQUENCE-START } +# block_mapping: { BLOCK-MAPPING-START } +# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY } +# indentless_sequence: { ENTRY } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_sequence: { FLOW-SEQUENCE-START } +# flow_mapping: { FLOW-MAPPING-START } +# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } +# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } + +__all__ = ['Parser', 'ParserError'] + +from error import MarkedYAMLError +from tokens import * +from events import * +from scanner import * + +class ParserError(MarkedYAMLError): + pass + +class Parser(object): + # Since writing a recursive-descendant parser is a straightforward task, we + # do not give many comments here. + + DEFAULT_TAGS = { + u'!': u'!', + u'!!': u'tag:yaml.org,2002:', + } + + def __init__(self): + self.current_event = None + self.yaml_version = None + self.tag_handles = {} + self.states = [] + self.marks = [] + self.state = self.parse_stream_start + + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + + def check_event(self, *choices): + # Check the type of the next event. + if self.current_event is None: + if self.state: + self.current_event = self.state() + if self.current_event is not None: + if not choices: + return True + for choice in choices: + if isinstance(self.current_event, choice): + return True + return False + + def peek_event(self): + # Get the next event. + if self.current_event is None: + if self.state: + self.current_event = self.state() + return self.current_event + + def get_event(self): + # Get the next event and proceed further. + if self.current_event is None: + if self.state: + self.current_event = self.state() + value = self.current_event + self.current_event = None + return value + + # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END + # implicit_document ::= block_node DOCUMENT-END* + # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* + + def parse_stream_start(self): + + # Parse the stream start. + token = self.get_token() + event = StreamStartEvent(token.start_mark, token.end_mark, + encoding=token.encoding) + + # Prepare the next state. + self.state = self.parse_implicit_document_start + + return event + + def parse_implicit_document_start(self): + + # Parse an implicit document. + if not self.check_token(DirectiveToken, DocumentStartToken, + StreamEndToken): + self.tag_handles = self.DEFAULT_TAGS + token = self.peek_token() + start_mark = end_mark = token.start_mark + event = DocumentStartEvent(start_mark, end_mark, + explicit=False) + + # Prepare the next state. + self.states.append(self.parse_document_end) + self.state = self.parse_block_node + + return event + + else: + return self.parse_document_start() + + def parse_document_start(self): + + # Parse any extra document end indicators. + while self.check_token(DocumentEndToken): + self.get_token() + + # Parse an explicit document. + if not self.check_token(StreamEndToken): + token = self.peek_token() + start_mark = token.start_mark + version, tags = self.process_directives() + if not self.check_token(DocumentStartToken): + raise ParserError(None, None, + "expected '', but found %r" + % self.peek_token().id, + self.peek_token().start_mark) + token = self.get_token() + end_mark = token.end_mark + event = DocumentStartEvent(start_mark, end_mark, + explicit=True, version=version, tags=tags) + self.states.append(self.parse_document_end) + self.state = self.parse_document_content + else: + # Parse the end of the stream. + token = self.get_token() + event = StreamEndEvent(token.start_mark, token.end_mark) + assert not self.states + assert not self.marks + self.state = None + return event + + def parse_document_end(self): + + # Parse the document end. + token = self.peek_token() + start_mark = end_mark = token.start_mark + explicit = False + if self.check_token(DocumentEndToken): + token = self.get_token() + end_mark = token.end_mark + explicit = True + event = DocumentEndEvent(start_mark, end_mark, + explicit=explicit) + + # Prepare the next state. + self.state = self.parse_document_start + + return event + + def parse_document_content(self): + if self.check_token(DirectiveToken, + DocumentStartToken, DocumentEndToken, StreamEndToken): + event = self.process_empty_scalar(self.peek_token().start_mark) + self.state = self.states.pop() + return event + else: + return self.parse_block_node() + + def process_directives(self): + self.yaml_version = None + self.tag_handles = {} + while self.check_token(DirectiveToken): + token = self.get_token() + if token.name == u'YAML': + if self.yaml_version is not None: + raise ParserError(None, None, + "found duplicate YAML directive", token.start_mark) + major, minor = token.value + if major != 1: + raise ParserError(None, None, + "found incompatible YAML document (version 1.* is required)", + token.start_mark) + self.yaml_version = token.value + elif token.name == u'TAG': + handle, prefix = token.value + if handle in self.tag_handles: + raise ParserError(None, None, + "duplicate tag handle %r" % handle.encode('utf-8'), + token.start_mark) + self.tag_handles[handle] = prefix + if self.tag_handles: + value = self.yaml_version, self.tag_handles.copy() + else: + value = self.yaml_version, None + for key in self.DEFAULT_TAGS: + if key not in self.tag_handles: + self.tag_handles[key] = self.DEFAULT_TAGS[key] + return value + + # block_node_or_indentless_sequence ::= ALIAS + # | properties (block_content | indentless_block_sequence)? + # | block_content + # | indentless_block_sequence + # block_node ::= ALIAS + # | properties block_content? + # | block_content + # flow_node ::= ALIAS + # | properties flow_content? + # | flow_content + # properties ::= TAG ANCHOR? | ANCHOR TAG? + # block_content ::= block_collection | flow_collection | SCALAR + # flow_content ::= flow_collection | SCALAR + # block_collection ::= block_sequence | block_mapping + # flow_collection ::= flow_sequence | flow_mapping + + def parse_block_node(self): + return self.parse_node(block=True) + + def parse_flow_node(self): + return self.parse_node() + + def parse_block_node_or_indentless_sequence(self): + return self.parse_node(block=True, indentless_sequence=True) + + def parse_node(self, block=False, indentless_sequence=False): + if self.check_token(AliasToken): + token = self.get_token() + event = AliasEvent(token.value, token.start_mark, token.end_mark) + self.state = self.states.pop() + else: + anchor = None + tag = None + start_mark = end_mark = tag_mark = None + if self.check_token(AnchorToken): + token = self.get_token() + start_mark = token.start_mark + end_mark = token.end_mark + anchor = token.value + if self.check_token(TagToken): + token = self.get_token() + tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + elif self.check_token(TagToken): + token = self.get_token() + start_mark = tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + if self.check_token(AnchorToken): + token = self.get_token() + end_mark = token.end_mark + anchor = token.value + if tag is not None: + handle, suffix = tag + if handle is not None: + if handle not in self.tag_handles: + raise ParserError("while parsing a node", start_mark, + "found undefined tag handle %r" % handle.encode('utf-8'), + tag_mark) + tag = self.tag_handles[handle]+suffix + else: + tag = suffix + #if tag == u'!': + # raise ParserError("while parsing a node", start_mark, + # "found non-specific tag '!'", tag_mark, + # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.") + if start_mark is None: + start_mark = end_mark = self.peek_token().start_mark + event = None + implicit = (tag is None or tag == u'!') + if indentless_sequence and self.check_token(BlockEntryToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark) + self.state = self.parse_indentless_sequence_entry + else: + if self.check_token(ScalarToken): + token = self.get_token() + end_mark = token.end_mark + if (token.plain and tag is None) or tag == u'!': + implicit = (True, False) + elif tag is None: + implicit = (False, True) + else: + implicit = (False, False) + event = ScalarEvent(anchor, tag, implicit, token.value, + start_mark, end_mark, style=token.style) + self.state = self.states.pop() + elif self.check_token(FlowSequenceStartToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + self.state = self.parse_flow_sequence_first_entry + elif self.check_token(FlowMappingStartToken): + end_mark = self.peek_token().end_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + self.state = self.parse_flow_mapping_first_key + elif block and self.check_token(BlockSequenceStartToken): + end_mark = self.peek_token().start_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + self.state = self.parse_block_sequence_first_entry + elif block and self.check_token(BlockMappingStartToken): + end_mark = self.peek_token().start_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + self.state = self.parse_block_mapping_first_key + elif anchor is not None or tag is not None: + # Empty scalars are allowed even if a tag or an anchor is + # specified. + event = ScalarEvent(anchor, tag, (implicit, False), u'', + start_mark, end_mark) + self.state = self.states.pop() + else: + if block: + node = 'block' + else: + node = 'flow' + token = self.peek_token() + raise ParserError("while parsing a %s node" % node, start_mark, + "expected the node content, but found %r" % token.id, + token.start_mark) + return event + + # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END + + def parse_block_sequence_first_entry(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_block_sequence_entry() + + def parse_block_sequence_entry(self): + if self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, BlockEndToken): + self.states.append(self.parse_block_sequence_entry) + return self.parse_block_node() + else: + self.state = self.parse_block_sequence_entry + return self.process_empty_scalar(token.end_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while parsing a block collection", self.marks[-1], + "expected , but found %r" % token.id, token.start_mark) + token = self.get_token() + event = SequenceEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + # indentless_sequence ::= (BLOCK-ENTRY block_node?)+ + + def parse_indentless_sequence_entry(self): + if self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, + KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_indentless_sequence_entry) + return self.parse_block_node() + else: + self.state = self.parse_indentless_sequence_entry + return self.process_empty_scalar(token.end_mark) + token = self.peek_token() + event = SequenceEndEvent(token.start_mark, token.start_mark) + self.state = self.states.pop() + return event + + # block_mapping ::= BLOCK-MAPPING_START + # ((KEY block_node_or_indentless_sequence?)? + # (VALUE block_node_or_indentless_sequence?)?)* + # BLOCK-END + + def parse_block_mapping_first_key(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_block_mapping_key() + + def parse_block_mapping_key(self): + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_block_mapping_value) + return self.parse_block_node_or_indentless_sequence() + else: + self.state = self.parse_block_mapping_value + return self.process_empty_scalar(token.end_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while parsing a block mapping", self.marks[-1], + "expected , but found %r" % token.id, token.start_mark) + token = self.get_token() + event = MappingEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_block_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_block_mapping_key) + return self.parse_block_node_or_indentless_sequence() + else: + self.state = self.parse_block_mapping_key + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_block_mapping_key + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + # flow_sequence ::= FLOW-SEQUENCE-START + # (flow_sequence_entry FLOW-ENTRY)* + # flow_sequence_entry? + # FLOW-SEQUENCE-END + # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + # + # Note that while production rules for both flow_sequence_entry and + # flow_mapping_entry are equal, their interpretations are different. + # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?` + # generate an inline mapping (set syntax). + + def parse_flow_sequence_first_entry(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_flow_sequence_entry(first=True) + + def parse_flow_sequence_entry(self, first=False): + if not self.check_token(FlowSequenceEndToken): + if not first: + if self.check_token(FlowEntryToken): + self.get_token() + else: + token = self.peek_token() + raise ParserError("while parsing a flow sequence", self.marks[-1], + "expected ',' or ']', but got %r" % token.id, token.start_mark) + + if self.check_token(KeyToken): + token = self.peek_token() + event = MappingStartEvent(None, None, True, + token.start_mark, token.end_mark, + flow_style=True) + self.state = self.parse_flow_sequence_entry_mapping_key + return event + elif not self.check_token(FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry) + return self.parse_flow_node() + token = self.get_token() + event = SequenceEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_flow_sequence_entry_mapping_key(self): + token = self.get_token() + if not self.check_token(ValueToken, + FlowEntryToken, FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry_mapping_value) + return self.parse_flow_node() + else: + self.state = self.parse_flow_sequence_entry_mapping_value + return self.process_empty_scalar(token.end_mark) + + def parse_flow_sequence_entry_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry_mapping_end) + return self.parse_flow_node() + else: + self.state = self.parse_flow_sequence_entry_mapping_end + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_flow_sequence_entry_mapping_end + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + def parse_flow_sequence_entry_mapping_end(self): + self.state = self.parse_flow_sequence_entry + token = self.peek_token() + return MappingEndEvent(token.start_mark, token.start_mark) + + # flow_mapping ::= FLOW-MAPPING-START + # (flow_mapping_entry FLOW-ENTRY)* + # flow_mapping_entry? + # FLOW-MAPPING-END + # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + + def parse_flow_mapping_first_key(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_flow_mapping_key(first=True) + + def parse_flow_mapping_key(self, first=False): + if not self.check_token(FlowMappingEndToken): + if not first: + if self.check_token(FlowEntryToken): + self.get_token() + else: + token = self.peek_token() + raise ParserError("while parsing a flow mapping", self.marks[-1], + "expected ',' or '}', but got %r" % token.id, token.start_mark) + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(ValueToken, + FlowEntryToken, FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_value) + return self.parse_flow_node() + else: + self.state = self.parse_flow_mapping_value + return self.process_empty_scalar(token.end_mark) + elif not self.check_token(FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_empty_value) + return self.parse_flow_node() + token = self.get_token() + event = MappingEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_flow_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_key) + return self.parse_flow_node() + else: + self.state = self.parse_flow_mapping_key + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_flow_mapping_key + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + def parse_flow_mapping_empty_value(self): + self.state = self.parse_flow_mapping_key + return self.process_empty_scalar(self.peek_token().start_mark) + + def process_empty_scalar(self, mark): + return ScalarEvent(None, None, (True, False), u'', mark, mark) + diff --git a/pipenv/patched/yaml2/reader.py b/pipenv/patched/yaml2/reader.py new file mode 100644 index 00000000..4b377d61 --- /dev/null +++ b/pipenv/patched/yaml2/reader.py @@ -0,0 +1,188 @@ +# This module contains abstractions for the input stream. You don't have to +# looks further, there are no pretty code. +# +# We define two classes here. +# +# Mark(source, line, column) +# It's just a record and its only use is producing nice error messages. +# Parser does not use it for any other purposes. +# +# Reader(source, data) +# Reader determines the encoding of `data` and converts it to unicode. +# Reader provides the following methods and attributes: +# reader.peek(length=1) - return the next `length` characters +# reader.forward(length=1) - move the current position to `length` characters. +# reader.index - the number of the current character. +# reader.line, stream.column - the line and the column of the current character. + +__all__ = ['Reader', 'ReaderError'] + +from error import YAMLError, Mark + +import codecs, re, sys + +has_ucs4 = sys.maxunicode > 0xffff + +class ReaderError(YAMLError): + + def __init__(self, name, position, character, encoding, reason): + self.name = name + self.character = character + self.position = position + self.encoding = encoding + self.reason = reason + + def __str__(self): + if isinstance(self.character, str): + return "'%s' codec can't decode byte #x%02x: %s\n" \ + " in \"%s\", position %d" \ + % (self.encoding, ord(self.character), self.reason, + self.name, self.position) + else: + return "unacceptable character #x%04x: %s\n" \ + " in \"%s\", position %d" \ + % (self.character, self.reason, + self.name, self.position) + +class Reader(object): + # Reader: + # - determines the data encoding and converts it to unicode, + # - checks if characters are in allowed range, + # - adds '\0' to the end. + + # Reader accepts + # - a `str` object, + # - a `unicode` object, + # - a file-like object with its `read` method returning `str`, + # - a file-like object with its `read` method returning `unicode`. + + # Yeah, it's ugly and slow. + + def __init__(self, stream): + self.name = None + self.stream = None + self.stream_pointer = 0 + self.eof = True + self.buffer = u'' + self.pointer = 0 + self.raw_buffer = None + self.raw_decode = None + self.encoding = None + self.index = 0 + self.line = 0 + self.column = 0 + if isinstance(stream, unicode): + self.name = "" + self.check_printable(stream) + self.buffer = stream+u'\0' + elif isinstance(stream, str): + self.name = "" + self.raw_buffer = stream + self.determine_encoding() + else: + self.stream = stream + self.name = getattr(stream, 'name', "") + self.eof = False + self.raw_buffer = '' + self.determine_encoding() + + def peek(self, index=0): + try: + return self.buffer[self.pointer+index] + except IndexError: + self.update(index+1) + return self.buffer[self.pointer+index] + + def prefix(self, length=1): + if self.pointer+length >= len(self.buffer): + self.update(length) + return self.buffer[self.pointer:self.pointer+length] + + def forward(self, length=1): + if self.pointer+length+1 >= len(self.buffer): + self.update(length+1) + while length: + ch = self.buffer[self.pointer] + self.pointer += 1 + self.index += 1 + if ch in u'\n\x85\u2028\u2029' \ + or (ch == u'\r' and self.buffer[self.pointer] != u'\n'): + self.line += 1 + self.column = 0 + elif ch != u'\uFEFF': + self.column += 1 + length -= 1 + + def get_mark(self): + if self.stream is None: + return Mark(self.name, self.index, self.line, self.column, + self.buffer, self.pointer) + else: + return Mark(self.name, self.index, self.line, self.column, + None, None) + + def determine_encoding(self): + while not self.eof and len(self.raw_buffer) < 2: + self.update_raw() + if not isinstance(self.raw_buffer, unicode): + if self.raw_buffer.startswith(codecs.BOM_UTF16_LE): + self.raw_decode = codecs.utf_16_le_decode + self.encoding = 'utf-16-le' + elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE): + self.raw_decode = codecs.utf_16_be_decode + self.encoding = 'utf-16-be' + else: + self.raw_decode = codecs.utf_8_decode + self.encoding = 'utf-8' + self.update(1) + + if has_ucs4: + NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]') + else: + NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uFFFD]|(?:^|[^\uD800-\uDBFF])[\uDC00-\uDFFF]|[\uD800-\uDBFF](?:[^\uDC00-\uDFFF]|$)') + def check_printable(self, data): + match = self.NON_PRINTABLE.search(data) + if match: + character = match.group() + position = self.index+(len(self.buffer)-self.pointer)+match.start() + raise ReaderError(self.name, position, ord(character), + 'unicode', "special characters are not allowed") + + def update(self, length): + if self.raw_buffer is None: + return + self.buffer = self.buffer[self.pointer:] + self.pointer = 0 + while len(self.buffer) < length: + if not self.eof: + self.update_raw() + if self.raw_decode is not None: + try: + data, converted = self.raw_decode(self.raw_buffer, + 'strict', self.eof) + except UnicodeDecodeError, exc: + character = exc.object[exc.start] + if self.stream is not None: + position = self.stream_pointer-len(self.raw_buffer)+exc.start + else: + position = exc.start + raise ReaderError(self.name, position, character, + exc.encoding, exc.reason) + else: + data = self.raw_buffer + converted = len(data) + self.check_printable(data) + self.buffer += data + self.raw_buffer = self.raw_buffer[converted:] + if self.eof: + self.buffer += u'\0' + self.raw_buffer = None + break + + def update_raw(self, size=1024): + data = self.stream.read(size) + if data: + self.raw_buffer += data + self.stream_pointer += len(data) + else: + self.eof = True diff --git a/pipenv/patched/yaml2/representer.py b/pipenv/patched/yaml2/representer.py new file mode 100644 index 00000000..93e09b67 --- /dev/null +++ b/pipenv/patched/yaml2/representer.py @@ -0,0 +1,489 @@ + +__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer', + 'RepresenterError'] + +from error import * + +from nodes import * + +import datetime + +import copy_reg, types + +class RepresenterError(YAMLError): + pass + +class BaseRepresenter(object): + + yaml_representers = {} + yaml_multi_representers = {} + + def __init__(self, default_style=None, default_flow_style=False, sort_keys=True): + self.default_style = default_style + self.default_flow_style = default_flow_style + self.sort_keys = sort_keys + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + + def represent(self, data): + node = self.represent_data(data) + self.serialize(node) + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + + def get_classobj_bases(self, cls): + bases = [cls] + for base in cls.__bases__: + bases.extend(self.get_classobj_bases(base)) + return bases + + def represent_data(self, data): + if self.ignore_aliases(data): + self.alias_key = None + else: + self.alias_key = id(data) + if self.alias_key is not None: + if self.alias_key in self.represented_objects: + node = self.represented_objects[self.alias_key] + #if node is None: + # raise RepresenterError("recursive objects are not allowed: %r" % data) + return node + #self.represented_objects[alias_key] = None + self.object_keeper.append(data) + data_types = type(data).__mro__ + if type(data) is types.InstanceType: + data_types = self.get_classobj_bases(data.__class__)+list(data_types) + if data_types[0] in self.yaml_representers: + node = self.yaml_representers[data_types[0]](self, data) + else: + for data_type in data_types: + if data_type in self.yaml_multi_representers: + node = self.yaml_multi_representers[data_type](self, data) + break + else: + if None in self.yaml_multi_representers: + node = self.yaml_multi_representers[None](self, data) + elif None in self.yaml_representers: + node = self.yaml_representers[None](self, data) + else: + node = ScalarNode(None, unicode(data)) + #if alias_key is not None: + # self.represented_objects[alias_key] = node + return node + + def add_representer(cls, data_type, representer): + if not 'yaml_representers' in cls.__dict__: + cls.yaml_representers = cls.yaml_representers.copy() + cls.yaml_representers[data_type] = representer + add_representer = classmethod(add_representer) + + def add_multi_representer(cls, data_type, representer): + if not 'yaml_multi_representers' in cls.__dict__: + cls.yaml_multi_representers = cls.yaml_multi_representers.copy() + cls.yaml_multi_representers[data_type] = representer + add_multi_representer = classmethod(add_multi_representer) + + def represent_scalar(self, tag, value, style=None): + if style is None: + style = self.default_style + node = ScalarNode(tag, value, style=style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + return node + + def represent_sequence(self, tag, sequence, flow_style=None): + value = [] + node = SequenceNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + for item in sequence: + node_item = self.represent_data(item) + if not (isinstance(node_item, ScalarNode) and not node_item.style): + best_style = False + value.append(node_item) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def represent_mapping(self, tag, mapping, flow_style=None): + value = [] + node = MappingNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + if hasattr(mapping, 'items'): + mapping = mapping.items() + if self.sort_keys: + mapping.sort() + for item_key, item_value in mapping: + node_key = self.represent_data(item_key) + node_value = self.represent_data(item_value) + if not (isinstance(node_key, ScalarNode) and not node_key.style): + best_style = False + if not (isinstance(node_value, ScalarNode) and not node_value.style): + best_style = False + value.append((node_key, node_value)) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def ignore_aliases(self, data): + return False + +class SafeRepresenter(BaseRepresenter): + + def ignore_aliases(self, data): + if data is None: + return True + if isinstance(data, tuple) and data == (): + return True + if isinstance(data, (str, unicode, bool, int, float)): + return True + + def represent_none(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:null', + u'null') + + def represent_str(self, data): + tag = None + style = None + try: + data = unicode(data, 'ascii') + tag = u'tag:yaml.org,2002:str' + except UnicodeDecodeError: + try: + data = unicode(data, 'utf-8') + tag = u'tag:yaml.org,2002:str' + except UnicodeDecodeError: + data = data.encode('base64') + tag = u'tag:yaml.org,2002:binary' + style = '|' + return self.represent_scalar(tag, data, style=style) + + def represent_unicode(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:str', data) + + def represent_bool(self, data): + if data: + value = u'true' + else: + value = u'false' + return self.represent_scalar(u'tag:yaml.org,2002:bool', value) + + def represent_int(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data)) + + def represent_long(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data)) + + inf_value = 1e300 + while repr(inf_value) != repr(inf_value*inf_value): + inf_value *= inf_value + + def represent_float(self, data): + if data != data or (data == 0.0 and data == 1.0): + value = u'.nan' + elif data == self.inf_value: + value = u'.inf' + elif data == -self.inf_value: + value = u'-.inf' + else: + value = unicode(repr(data)).lower() + # Note that in some cases `repr(data)` represents a float number + # without the decimal parts. For instance: + # >>> repr(1e17) + # '1e17' + # Unfortunately, this is not a valid float representation according + # to the definition of the `!!float` tag. We fix this by adding + # '.0' before the 'e' symbol. + if u'.' not in value and u'e' in value: + value = value.replace(u'e', u'.0e', 1) + return self.represent_scalar(u'tag:yaml.org,2002:float', value) + + def represent_list(self, data): + #pairs = (len(data) > 0 and isinstance(data, list)) + #if pairs: + # for item in data: + # if not isinstance(item, tuple) or len(item) != 2: + # pairs = False + # break + #if not pairs: + return self.represent_sequence(u'tag:yaml.org,2002:seq', data) + #value = [] + #for item_key, item_value in data: + # value.append(self.represent_mapping(u'tag:yaml.org,2002:map', + # [(item_key, item_value)])) + #return SequenceNode(u'tag:yaml.org,2002:pairs', value) + + def represent_dict(self, data): + return self.represent_mapping(u'tag:yaml.org,2002:map', data) + + def represent_set(self, data): + value = {} + for key in data: + value[key] = None + return self.represent_mapping(u'tag:yaml.org,2002:set', value) + + def represent_date(self, data): + value = unicode(data.isoformat()) + return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value) + + def represent_datetime(self, data): + value = unicode(data.isoformat(' ')) + return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value) + + def represent_yaml_object(self, tag, data, cls, flow_style=None): + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__.copy() + return self.represent_mapping(tag, state, flow_style=flow_style) + + def represent_undefined(self, data): + raise RepresenterError("cannot represent an object", data) + +SafeRepresenter.add_representer(type(None), + SafeRepresenter.represent_none) + +SafeRepresenter.add_representer(str, + SafeRepresenter.represent_str) + +SafeRepresenter.add_representer(unicode, + SafeRepresenter.represent_unicode) + +SafeRepresenter.add_representer(bool, + SafeRepresenter.represent_bool) + +SafeRepresenter.add_representer(int, + SafeRepresenter.represent_int) + +SafeRepresenter.add_representer(long, + SafeRepresenter.represent_long) + +SafeRepresenter.add_representer(float, + SafeRepresenter.represent_float) + +SafeRepresenter.add_representer(list, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(tuple, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(dict, + SafeRepresenter.represent_dict) + +SafeRepresenter.add_representer(set, + SafeRepresenter.represent_set) + +SafeRepresenter.add_representer(datetime.date, + SafeRepresenter.represent_date) + +SafeRepresenter.add_representer(datetime.datetime, + SafeRepresenter.represent_datetime) + +SafeRepresenter.add_representer(None, + SafeRepresenter.represent_undefined) + +class Representer(SafeRepresenter): + + def represent_str(self, data): + tag = None + style = None + try: + data = unicode(data, 'ascii') + tag = u'tag:yaml.org,2002:str' + except UnicodeDecodeError: + try: + data = unicode(data, 'utf-8') + tag = u'tag:yaml.org,2002:python/str' + except UnicodeDecodeError: + data = data.encode('base64') + tag = u'tag:yaml.org,2002:binary' + style = '|' + return self.represent_scalar(tag, data, style=style) + + def represent_unicode(self, data): + tag = None + try: + data.encode('ascii') + tag = u'tag:yaml.org,2002:python/unicode' + except UnicodeEncodeError: + tag = u'tag:yaml.org,2002:str' + return self.represent_scalar(tag, data) + + def represent_long(self, data): + tag = u'tag:yaml.org,2002:int' + if int(data) is not data: + tag = u'tag:yaml.org,2002:python/long' + return self.represent_scalar(tag, unicode(data)) + + def represent_complex(self, data): + if data.imag == 0.0: + data = u'%r' % data.real + elif data.real == 0.0: + data = u'%rj' % data.imag + elif data.imag > 0: + data = u'%r+%rj' % (data.real, data.imag) + else: + data = u'%r%rj' % (data.real, data.imag) + return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data) + + def represent_tuple(self, data): + return self.represent_sequence(u'tag:yaml.org,2002:python/tuple', data) + + def represent_name(self, data): + name = u'%s.%s' % (data.__module__, data.__name__) + return self.represent_scalar(u'tag:yaml.org,2002:python/name:'+name, u'') + + def represent_module(self, data): + return self.represent_scalar( + u'tag:yaml.org,2002:python/module:'+data.__name__, u'') + + def represent_instance(self, data): + # For instances of classic classes, we use __getinitargs__ and + # __getstate__ to serialize the data. + + # If data.__getinitargs__ exists, the object must be reconstructed by + # calling cls(**args), where args is a tuple returned by + # __getinitargs__. Otherwise, the cls.__init__ method should never be + # called and the class instance is created by instantiating a trivial + # class and assigning to the instance's __class__ variable. + + # If data.__getstate__ exists, it returns the state of the object. + # Otherwise, the state of the object is data.__dict__. + + # We produce either a !!python/object or !!python/object/new node. + # If data.__getinitargs__ does not exist and state is a dictionary, we + # produce a !!python/object node . Otherwise we produce a + # !!python/object/new node. + + cls = data.__class__ + class_name = u'%s.%s' % (cls.__module__, cls.__name__) + args = None + state = None + if hasattr(data, '__getinitargs__'): + args = list(data.__getinitargs__()) + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__ + if args is None and isinstance(state, dict): + return self.represent_mapping( + u'tag:yaml.org,2002:python/object:'+class_name, state) + if isinstance(state, dict) and not state: + return self.represent_sequence( + u'tag:yaml.org,2002:python/object/new:'+class_name, args) + value = {} + if args: + value['args'] = args + value['state'] = state + return self.represent_mapping( + u'tag:yaml.org,2002:python/object/new:'+class_name, value) + + def represent_object(self, data): + # We use __reduce__ API to save the data. data.__reduce__ returns + # a tuple of length 2-5: + # (function, args, state, listitems, dictitems) + + # For reconstructing, we calls function(*args), then set its state, + # listitems, and dictitems if they are not None. + + # A special case is when function.__name__ == '__newobj__'. In this + # case we create the object with args[0].__new__(*args). + + # Another special case is when __reduce__ returns a string - we don't + # support it. + + # We produce a !!python/object, !!python/object/new or + # !!python/object/apply node. + + cls = type(data) + if cls in copy_reg.dispatch_table: + reduce = copy_reg.dispatch_table[cls](data) + elif hasattr(data, '__reduce_ex__'): + reduce = data.__reduce_ex__(2) + elif hasattr(data, '__reduce__'): + reduce = data.__reduce__() + else: + raise RepresenterError("cannot represent an object", data) + reduce = (list(reduce)+[None]*5)[:5] + function, args, state, listitems, dictitems = reduce + args = list(args) + if state is None: + state = {} + if listitems is not None: + listitems = list(listitems) + if dictitems is not None: + dictitems = dict(dictitems) + if function.__name__ == '__newobj__': + function = args[0] + args = args[1:] + tag = u'tag:yaml.org,2002:python/object/new:' + newobj = True + else: + tag = u'tag:yaml.org,2002:python/object/apply:' + newobj = False + function_name = u'%s.%s' % (function.__module__, function.__name__) + if not args and not listitems and not dictitems \ + and isinstance(state, dict) and newobj: + return self.represent_mapping( + u'tag:yaml.org,2002:python/object:'+function_name, state) + if not listitems and not dictitems \ + and isinstance(state, dict) and not state: + return self.represent_sequence(tag+function_name, args) + value = {} + if args: + value['args'] = args + if state or not isinstance(state, dict): + value['state'] = state + if listitems: + value['listitems'] = listitems + if dictitems: + value['dictitems'] = dictitems + return self.represent_mapping(tag+function_name, value) + +Representer.add_representer(str, + Representer.represent_str) + +Representer.add_representer(unicode, + Representer.represent_unicode) + +Representer.add_representer(long, + Representer.represent_long) + +Representer.add_representer(complex, + Representer.represent_complex) + +Representer.add_representer(tuple, + Representer.represent_tuple) + +Representer.add_representer(type, + Representer.represent_name) + +Representer.add_representer(types.ClassType, + Representer.represent_name) + +Representer.add_representer(types.FunctionType, + Representer.represent_name) + +Representer.add_representer(types.BuiltinFunctionType, + Representer.represent_name) + +Representer.add_representer(types.ModuleType, + Representer.represent_module) + +Representer.add_multi_representer(types.InstanceType, + Representer.represent_instance) + +Representer.add_multi_representer(object, + Representer.represent_object) + diff --git a/pipenv/patched/yaml2/resolver.py b/pipenv/patched/yaml2/resolver.py new file mode 100644 index 00000000..528fbc0e --- /dev/null +++ b/pipenv/patched/yaml2/resolver.py @@ -0,0 +1,227 @@ + +__all__ = ['BaseResolver', 'Resolver'] + +from error import * +from nodes import * + +import re + +class ResolverError(YAMLError): + pass + +class BaseResolver(object): + + DEFAULT_SCALAR_TAG = u'tag:yaml.org,2002:str' + DEFAULT_SEQUENCE_TAG = u'tag:yaml.org,2002:seq' + DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map' + + yaml_implicit_resolvers = {} + yaml_path_resolvers = {} + + def __init__(self): + self.resolver_exact_paths = [] + self.resolver_prefix_paths = [] + + def add_implicit_resolver(cls, tag, regexp, first): + if not 'yaml_implicit_resolvers' in cls.__dict__: + implicit_resolvers = {} + for key in cls.yaml_implicit_resolvers: + implicit_resolvers[key] = cls.yaml_implicit_resolvers[key][:] + cls.yaml_implicit_resolvers = implicit_resolvers + if first is None: + first = [None] + for ch in first: + cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp)) + add_implicit_resolver = classmethod(add_implicit_resolver) + + def add_path_resolver(cls, tag, path, kind=None): + # Note: `add_path_resolver` is experimental. The API could be changed. + # `new_path` is a pattern that is matched against the path from the + # root to the node that is being considered. `node_path` elements are + # tuples `(node_check, index_check)`. `node_check` is a node class: + # `ScalarNode`, `SequenceNode`, `MappingNode` or `None`. `None` + # matches any kind of a node. `index_check` could be `None`, a boolean + # value, a string value, or a number. `None` and `False` match against + # any _value_ of sequence and mapping nodes. `True` matches against + # any _key_ of a mapping node. A string `index_check` matches against + # a mapping value that corresponds to a scalar key which content is + # equal to the `index_check` value. An integer `index_check` matches + # against a sequence value with the index equal to `index_check`. + if not 'yaml_path_resolvers' in cls.__dict__: + cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy() + new_path = [] + for element in path: + if isinstance(element, (list, tuple)): + if len(element) == 2: + node_check, index_check = element + elif len(element) == 1: + node_check = element[0] + index_check = True + else: + raise ResolverError("Invalid path element: %s" % element) + else: + node_check = None + index_check = element + if node_check is str: + node_check = ScalarNode + elif node_check is list: + node_check = SequenceNode + elif node_check is dict: + node_check = MappingNode + elif node_check not in [ScalarNode, SequenceNode, MappingNode] \ + and not isinstance(node_check, basestring) \ + and node_check is not None: + raise ResolverError("Invalid node checker: %s" % node_check) + if not isinstance(index_check, (basestring, int)) \ + and index_check is not None: + raise ResolverError("Invalid index checker: %s" % index_check) + new_path.append((node_check, index_check)) + if kind is str: + kind = ScalarNode + elif kind is list: + kind = SequenceNode + elif kind is dict: + kind = MappingNode + elif kind not in [ScalarNode, SequenceNode, MappingNode] \ + and kind is not None: + raise ResolverError("Invalid node kind: %s" % kind) + cls.yaml_path_resolvers[tuple(new_path), kind] = tag + add_path_resolver = classmethod(add_path_resolver) + + def descend_resolver(self, current_node, current_index): + if not self.yaml_path_resolvers: + return + exact_paths = {} + prefix_paths = [] + if current_node: + depth = len(self.resolver_prefix_paths) + for path, kind in self.resolver_prefix_paths[-1]: + if self.check_resolver_prefix(depth, path, kind, + current_node, current_index): + if len(path) > depth: + prefix_paths.append((path, kind)) + else: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + for path, kind in self.yaml_path_resolvers: + if not path: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + prefix_paths.append((path, kind)) + self.resolver_exact_paths.append(exact_paths) + self.resolver_prefix_paths.append(prefix_paths) + + def ascend_resolver(self): + if not self.yaml_path_resolvers: + return + self.resolver_exact_paths.pop() + self.resolver_prefix_paths.pop() + + def check_resolver_prefix(self, depth, path, kind, + current_node, current_index): + node_check, index_check = path[depth-1] + if isinstance(node_check, basestring): + if current_node.tag != node_check: + return + elif node_check is not None: + if not isinstance(current_node, node_check): + return + if index_check is True and current_index is not None: + return + if (index_check is False or index_check is None) \ + and current_index is None: + return + if isinstance(index_check, basestring): + if not (isinstance(current_index, ScalarNode) + and index_check == current_index.value): + return + elif isinstance(index_check, int) and not isinstance(index_check, bool): + if index_check != current_index: + return + return True + + def resolve(self, kind, value, implicit): + if kind is ScalarNode and implicit[0]: + if value == u'': + resolvers = self.yaml_implicit_resolvers.get(u'', []) + else: + resolvers = self.yaml_implicit_resolvers.get(value[0], []) + resolvers += self.yaml_implicit_resolvers.get(None, []) + for tag, regexp in resolvers: + if regexp.match(value): + return tag + implicit = implicit[1] + if self.yaml_path_resolvers: + exact_paths = self.resolver_exact_paths[-1] + if kind in exact_paths: + return exact_paths[kind] + if None in exact_paths: + return exact_paths[None] + if kind is ScalarNode: + return self.DEFAULT_SCALAR_TAG + elif kind is SequenceNode: + return self.DEFAULT_SEQUENCE_TAG + elif kind is MappingNode: + return self.DEFAULT_MAPPING_TAG + +class Resolver(BaseResolver): + pass + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:bool', + re.compile(ur'''^(?:yes|Yes|YES|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF)$''', re.X), + list(u'yYnNtTfFoO')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:float', + re.compile(ur'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? + |\.[0-9_]+(?:[eE][-+][0-9]+)? + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]* + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$''', re.X), + list(u'-+0123456789.')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:int', + re.compile(ur'''^(?:[-+]?0b[0-1_]+ + |[-+]?0[0-7_]+ + |[-+]?(?:0|[1-9][0-9_]*) + |[-+]?0x[0-9a-fA-F_]+ + |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), + list(u'-+0123456789')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:merge', + re.compile(ur'^(?:<<)$'), + [u'<']) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:null', + re.compile(ur'''^(?: ~ + |null|Null|NULL + | )$''', re.X), + [u'~', u'n', u'N', u'']) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:timestamp', + re.compile(ur'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] + |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? + (?:[Tt]|[ \t]+)[0-9][0-9]? + :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)? + (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), + list(u'0123456789')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:value', + re.compile(ur'^(?:=)$'), + [u'=']) + +# The following resolver is only for documentation purposes. It cannot work +# because plain scalars cannot start with '!', '&', or '*'. +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:yaml', + re.compile(ur'^(?:!|&|\*)$'), + list(u'!&*')) + diff --git a/pipenv/patched/yaml2/scanner.py b/pipenv/patched/yaml2/scanner.py new file mode 100644 index 00000000..098ea7be --- /dev/null +++ b/pipenv/patched/yaml2/scanner.py @@ -0,0 +1,1444 @@ + +# Scanner produces tokens of the following types: +# STREAM-START +# STREAM-END +# DIRECTIVE(name, value) +# DOCUMENT-START +# DOCUMENT-END +# BLOCK-SEQUENCE-START +# BLOCK-MAPPING-START +# BLOCK-END +# FLOW-SEQUENCE-START +# FLOW-MAPPING-START +# FLOW-SEQUENCE-END +# FLOW-MAPPING-END +# BLOCK-ENTRY +# FLOW-ENTRY +# KEY +# VALUE +# ALIAS(value) +# ANCHOR(value) +# TAG(value) +# SCALAR(value, plain, style) +# +# Read comments in the Scanner code for more details. +# + +__all__ = ['Scanner', 'ScannerError'] + +from error import MarkedYAMLError +from tokens import * + +class ScannerError(MarkedYAMLError): + pass + +class SimpleKey(object): + # See below simple keys treatment. + + def __init__(self, token_number, required, index, line, column, mark): + self.token_number = token_number + self.required = required + self.index = index + self.line = line + self.column = column + self.mark = mark + +class Scanner(object): + + def __init__(self): + """Initialize the scanner.""" + # It is assumed that Scanner and Reader will have a common descendant. + # Reader do the dirty work of checking for BOM and converting the + # input data to Unicode. It also adds NUL to the end. + # + # Reader supports the following methods + # self.peek(i=0) # peek the next i-th character + # self.prefix(l=1) # peek the next l characters + # self.forward(l=1) # read the next l characters and move the pointer. + + # Had we reached the end of the stream? + self.done = False + + # The number of unclosed '{' and '['. `flow_level == 0` means block + # context. + self.flow_level = 0 + + # List of processed tokens that are not yet emitted. + self.tokens = [] + + # Add the STREAM-START token. + self.fetch_stream_start() + + # Number of tokens that were emitted through the `get_token` method. + self.tokens_taken = 0 + + # The current indentation level. + self.indent = -1 + + # Past indentation levels. + self.indents = [] + + # Variables related to simple keys treatment. + + # A simple key is a key that is not denoted by the '?' indicator. + # Example of simple keys: + # --- + # block simple key: value + # ? not a simple key: + # : { flow simple key: value } + # We emit the KEY token before all keys, so when we find a potential + # simple key, we try to locate the corresponding ':' indicator. + # Simple keys should be limited to a single line and 1024 characters. + + # Can a simple key start at the current position? A simple key may + # start: + # - at the beginning of the line, not counting indentation spaces + # (in block context), + # - after '{', '[', ',' (in the flow context), + # - after '?', ':', '-' (in the block context). + # In the block context, this flag also signifies if a block collection + # may start at the current position. + self.allow_simple_key = True + + # Keep track of possible simple keys. This is a dictionary. The key + # is `flow_level`; there can be no more that one possible simple key + # for each level. The value is a SimpleKey record: + # (token_number, required, index, line, column, mark) + # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow), + # '[', or '{' tokens. + self.possible_simple_keys = {} + + # Public methods. + + def check_token(self, *choices): + # Check if the next token is one of the given types. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + if not choices: + return True + for choice in choices: + if isinstance(self.tokens[0], choice): + return True + return False + + def peek_token(self): + # Return the next token, but do not delete if from the queue. + # Return None if no more tokens. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + return self.tokens[0] + else: + return None + + def get_token(self): + # Return the next token. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + self.tokens_taken += 1 + return self.tokens.pop(0) + + # Private methods. + + def need_more_tokens(self): + if self.done: + return False + if not self.tokens: + return True + # The current token may be a potential simple key, so we + # need to look further. + self.stale_possible_simple_keys() + if self.next_possible_simple_key() == self.tokens_taken: + return True + + def fetch_more_tokens(self): + + # Eat whitespaces and comments until we reach the next token. + self.scan_to_next_token() + + # Remove obsolete possible simple keys. + self.stale_possible_simple_keys() + + # Compare the current indentation and column. It may add some tokens + # and decrease the current indentation level. + self.unwind_indent(self.column) + + # Peek the next character. + ch = self.peek() + + # Is it the end of stream? + if ch == u'\0': + return self.fetch_stream_end() + + # Is it a directive? + if ch == u'%' and self.check_directive(): + return self.fetch_directive() + + # Is it the document start? + if ch == u'-' and self.check_document_start(): + return self.fetch_document_start() + + # Is it the document end? + if ch == u'.' and self.check_document_end(): + return self.fetch_document_end() + + # TODO: support for BOM within a stream. + #if ch == u'\uFEFF': + # return self.fetch_bom() <-- issue BOMToken + + # Note: the order of the following checks is NOT significant. + + # Is it the flow sequence start indicator? + if ch == u'[': + return self.fetch_flow_sequence_start() + + # Is it the flow mapping start indicator? + if ch == u'{': + return self.fetch_flow_mapping_start() + + # Is it the flow sequence end indicator? + if ch == u']': + return self.fetch_flow_sequence_end() + + # Is it the flow mapping end indicator? + if ch == u'}': + return self.fetch_flow_mapping_end() + + # Is it the flow entry indicator? + if ch == u',': + return self.fetch_flow_entry() + + # Is it the block entry indicator? + if ch == u'-' and self.check_block_entry(): + return self.fetch_block_entry() + + # Is it the key indicator? + if ch == u'?' and self.check_key(): + return self.fetch_key() + + # Is it the value indicator? + if ch == u':' and self.check_value(): + return self.fetch_value() + + # Is it an alias? + if ch == u'*': + return self.fetch_alias() + + # Is it an anchor? + if ch == u'&': + return self.fetch_anchor() + + # Is it a tag? + if ch == u'!': + return self.fetch_tag() + + # Is it a literal scalar? + if ch == u'|' and not self.flow_level: + return self.fetch_literal() + + # Is it a folded scalar? + if ch == u'>' and not self.flow_level: + return self.fetch_folded() + + # Is it a single quoted scalar? + if ch == u'\'': + return self.fetch_single() + + # Is it a double quoted scalar? + if ch == u'\"': + return self.fetch_double() + + # It must be a plain scalar then. + if self.check_plain(): + return self.fetch_plain() + + # No? It's an error. Let's produce a nice error message. + raise ScannerError("while scanning for the next token", None, + "found character %r that cannot start any token" + % ch.encode('utf-8'), self.get_mark()) + + # Simple keys treatment. + + def next_possible_simple_key(self): + # Return the number of the nearest possible simple key. Actually we + # don't need to loop through the whole dictionary. We may replace it + # with the following code: + # if not self.possible_simple_keys: + # return None + # return self.possible_simple_keys[ + # min(self.possible_simple_keys.keys())].token_number + min_token_number = None + for level in self.possible_simple_keys: + key = self.possible_simple_keys[level] + if min_token_number is None or key.token_number < min_token_number: + min_token_number = key.token_number + return min_token_number + + def stale_possible_simple_keys(self): + # Remove entries that are no longer possible simple keys. According to + # the YAML specification, simple keys + # - should be limited to a single line, + # - should be no longer than 1024 characters. + # Disabling this procedure will allow simple keys of any length and + # height (may cause problems if indentation is broken though). + for level in self.possible_simple_keys.keys(): + key = self.possible_simple_keys[level] + if key.line != self.line \ + or self.index-key.index > 1024: + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self.get_mark()) + del self.possible_simple_keys[level] + + def save_possible_simple_key(self): + # The next token may start a simple key. We check if it's possible + # and save its position. This function is called for + # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'. + + # Check if a simple key is required at the current position. + required = not self.flow_level and self.indent == self.column + + # The next token might be a simple key. Let's save it's number and + # position. + if self.allow_simple_key: + self.remove_possible_simple_key() + token_number = self.tokens_taken+len(self.tokens) + key = SimpleKey(token_number, required, + self.index, self.line, self.column, self.get_mark()) + self.possible_simple_keys[self.flow_level] = key + + def remove_possible_simple_key(self): + # Remove the saved possible key position at the current flow level. + if self.flow_level in self.possible_simple_keys: + key = self.possible_simple_keys[self.flow_level] + + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self.get_mark()) + + del self.possible_simple_keys[self.flow_level] + + # Indentation functions. + + def unwind_indent(self, column): + + ## In flow context, tokens should respect indentation. + ## Actually the condition should be `self.indent >= column` according to + ## the spec. But this condition will prohibit intuitively correct + ## constructions such as + ## key : { + ## } + #if self.flow_level and self.indent > column: + # raise ScannerError(None, None, + # "invalid indentation or unclosed '[' or '{'", + # self.get_mark()) + + # In the flow context, indentation is ignored. We make the scanner less + # restrictive then specification requires. + if self.flow_level: + return + + # In block context, we may need to issue the BLOCK-END tokens. + while self.indent > column: + mark = self.get_mark() + self.indent = self.indents.pop() + self.tokens.append(BlockEndToken(mark, mark)) + + def add_indent(self, column): + # Check if we need to increase indentation. + if self.indent < column: + self.indents.append(self.indent) + self.indent = column + return True + return False + + # Fetchers. + + def fetch_stream_start(self): + # We always add STREAM-START as the first token and STREAM-END as the + # last token. + + # Read the token. + mark = self.get_mark() + + # Add STREAM-START. + self.tokens.append(StreamStartToken(mark, mark, + encoding=self.encoding)) + + + def fetch_stream_end(self): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. + self.remove_possible_simple_key() + self.allow_simple_key = False + self.possible_simple_keys = {} + + # Read the token. + mark = self.get_mark() + + # Add STREAM-END. + self.tokens.append(StreamEndToken(mark, mark)) + + # The steam is finished. + self.done = True + + def fetch_directive(self): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Scan and add DIRECTIVE. + self.tokens.append(self.scan_directive()) + + def fetch_document_start(self): + self.fetch_document_indicator(DocumentStartToken) + + def fetch_document_end(self): + self.fetch_document_indicator(DocumentEndToken) + + def fetch_document_indicator(self, TokenClass): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. Note that there could not be a block collection + # after '---'. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Add DOCUMENT-START or DOCUMENT-END. + start_mark = self.get_mark() + self.forward(3) + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_start(self): + self.fetch_flow_collection_start(FlowSequenceStartToken) + + def fetch_flow_mapping_start(self): + self.fetch_flow_collection_start(FlowMappingStartToken) + + def fetch_flow_collection_start(self, TokenClass): + + # '[' and '{' may start a simple key. + self.save_possible_simple_key() + + # Increase the flow level. + self.flow_level += 1 + + # Simple keys are allowed after '[' and '{'. + self.allow_simple_key = True + + # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_end(self): + self.fetch_flow_collection_end(FlowSequenceEndToken) + + def fetch_flow_mapping_end(self): + self.fetch_flow_collection_end(FlowMappingEndToken) + + def fetch_flow_collection_end(self, TokenClass): + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Decrease the flow level. + self.flow_level -= 1 + + # No simple keys after ']' or '}'. + self.allow_simple_key = False + + # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_entry(self): + + # Simple keys are allowed after ','. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add FLOW-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(FlowEntryToken(start_mark, end_mark)) + + def fetch_block_entry(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a new entry? + if not self.allow_simple_key: + raise ScannerError(None, None, + "sequence entries are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-SEQUENCE-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockSequenceStartToken(mark, mark)) + + # It's an error for the block entry to occur in the flow context, + # but we let the parser detect this. + else: + pass + + # Simple keys are allowed after '-'. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add BLOCK-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(BlockEntryToken(start_mark, end_mark)) + + def fetch_key(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a key (not necessary a simple)? + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping keys are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-MAPPING-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockMappingStartToken(mark, mark)) + + # Simple keys are allowed after '?' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add KEY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(KeyToken(start_mark, end_mark)) + + def fetch_value(self): + + # Do we determine a simple key? + if self.flow_level in self.possible_simple_keys: + + # Add KEY. + key = self.possible_simple_keys[self.flow_level] + del self.possible_simple_keys[self.flow_level] + self.tokens.insert(key.token_number-self.tokens_taken, + KeyToken(key.mark, key.mark)) + + # If this key starts a new block mapping, we need to add + # BLOCK-MAPPING-START. + if not self.flow_level: + if self.add_indent(key.column): + self.tokens.insert(key.token_number-self.tokens_taken, + BlockMappingStartToken(key.mark, key.mark)) + + # There cannot be two simple keys one after another. + self.allow_simple_key = False + + # It must be a part of a complex key. + else: + + # Block context needs additional checks. + # (Do we really need them? They will be caught by the parser + # anyway.) + if not self.flow_level: + + # We are allowed to start a complex value if and only if + # we can start a simple key. + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping values are not allowed here", + self.get_mark()) + + # If this value starts a new block mapping, we need to add + # BLOCK-MAPPING-START. It will be detected as an error later by + # the parser. + if not self.flow_level: + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockMappingStartToken(mark, mark)) + + # Simple keys are allowed after ':' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add VALUE. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(ValueToken(start_mark, end_mark)) + + def fetch_alias(self): + + # ALIAS could be a simple key. + self.save_possible_simple_key() + + # No simple keys after ALIAS. + self.allow_simple_key = False + + # Scan and add ALIAS. + self.tokens.append(self.scan_anchor(AliasToken)) + + def fetch_anchor(self): + + # ANCHOR could start a simple key. + self.save_possible_simple_key() + + # No simple keys after ANCHOR. + self.allow_simple_key = False + + # Scan and add ANCHOR. + self.tokens.append(self.scan_anchor(AnchorToken)) + + def fetch_tag(self): + + # TAG could start a simple key. + self.save_possible_simple_key() + + # No simple keys after TAG. + self.allow_simple_key = False + + # Scan and add TAG. + self.tokens.append(self.scan_tag()) + + def fetch_literal(self): + self.fetch_block_scalar(style='|') + + def fetch_folded(self): + self.fetch_block_scalar(style='>') + + def fetch_block_scalar(self, style): + + # A simple key may follow a block scalar. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Scan and add SCALAR. + self.tokens.append(self.scan_block_scalar(style)) + + def fetch_single(self): + self.fetch_flow_scalar(style='\'') + + def fetch_double(self): + self.fetch_flow_scalar(style='"') + + def fetch_flow_scalar(self, style): + + # A flow scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after flow scalars. + self.allow_simple_key = False + + # Scan and add SCALAR. + self.tokens.append(self.scan_flow_scalar(style)) + + def fetch_plain(self): + + # A plain scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after plain scalars. But note that `scan_plain` will + # change this flag if the scan is finished at the beginning of the + # line. + self.allow_simple_key = False + + # Scan and add SCALAR. May change `allow_simple_key`. + self.tokens.append(self.scan_plain()) + + # Checkers. + + def check_directive(self): + + # DIRECTIVE: ^ '%' ... + # The '%' indicator is already checked. + if self.column == 0: + return True + + def check_document_start(self): + + # DOCUMENT-START: ^ '---' (' '|'\n') + if self.column == 0: + if self.prefix(3) == u'---' \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return True + + def check_document_end(self): + + # DOCUMENT-END: ^ '...' (' '|'\n') + if self.column == 0: + if self.prefix(3) == u'...' \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return True + + def check_block_entry(self): + + # BLOCK-ENTRY: '-' (' '|'\n') + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + + def check_key(self): + + # KEY(flow context): '?' + if self.flow_level: + return True + + # KEY(block context): '?' (' '|'\n') + else: + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + + def check_value(self): + + # VALUE(flow context): ':' + if self.flow_level: + return True + + # VALUE(block context): ':' (' '|'\n') + else: + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + + def check_plain(self): + + # A plain scalar may start with any non-space character except: + # '-', '?', ':', ',', '[', ']', '{', '}', + # '#', '&', '*', '!', '|', '>', '\'', '\"', + # '%', '@', '`'. + # + # It may also start with + # '-', '?', ':' + # if it is followed by a non-space character. + # + # Note that we limit the last rule to the block context (except the + # '-' character) because we want the flow context to be space + # independent. + ch = self.peek() + return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \ + or (self.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029' + and (ch == u'-' or (not self.flow_level and ch in u'?:'))) + + # Scanners. + + def scan_to_next_token(self): + # We ignore spaces, line breaks and comments. + # If we find a line break in the block context, we set the flag + # `allow_simple_key` on. + # The byte order mark is stripped if it's the first character in the + # stream. We do not yet support BOM inside the stream as the + # specification requires. Any such mark will be considered as a part + # of the document. + # + # TODO: We need to make tab handling rules more sane. A good rule is + # Tabs cannot precede tokens + # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END, + # KEY(block), VALUE(block), BLOCK-ENTRY + # So the checking code is + # if : + # self.allow_simple_keys = False + # We also need to add the check for `allow_simple_keys == True` to + # `unwind_indent` before issuing BLOCK-END. + # Scanners for block, flow, and plain scalars need to be modified. + + if self.index == 0 and self.peek() == u'\uFEFF': + self.forward() + found = False + while not found: + while self.peek() == u' ': + self.forward() + if self.peek() == u'#': + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + if self.scan_line_break(): + if not self.flow_level: + self.allow_simple_key = True + else: + found = True + + def scan_directive(self): + # See the specification for details. + start_mark = self.get_mark() + self.forward() + name = self.scan_directive_name(start_mark) + value = None + if name == u'YAML': + value = self.scan_yaml_directive_value(start_mark) + end_mark = self.get_mark() + elif name == u'TAG': + value = self.scan_tag_directive_value(start_mark) + end_mark = self.get_mark() + else: + end_mark = self.get_mark() + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + self.scan_directive_ignored_line(start_mark) + return DirectiveToken(name, value, start_mark, end_mark) + + def scan_directive_name(self, start_mark): + # See the specification for details. + length = 0 + ch = self.peek(length) + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + return value + + def scan_yaml_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + major = self.scan_yaml_directive_number(start_mark) + if self.peek() != '.': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or '.', but found %r" + % self.peek().encode('utf-8'), + self.get_mark()) + self.forward() + minor = self.scan_yaml_directive_number(start_mark) + if self.peek() not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or ' ', but found %r" + % self.peek().encode('utf-8'), + self.get_mark()) + return (major, minor) + + def scan_yaml_directive_number(self, start_mark): + # See the specification for details. + ch = self.peek() + if not (u'0' <= ch <= u'9'): + raise ScannerError("while scanning a directive", start_mark, + "expected a digit, but found %r" % ch.encode('utf-8'), + self.get_mark()) + length = 0 + while u'0' <= self.peek(length) <= u'9': + length += 1 + value = int(self.prefix(length)) + self.forward(length) + return value + + def scan_tag_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + handle = self.scan_tag_directive_handle(start_mark) + while self.peek() == u' ': + self.forward() + prefix = self.scan_tag_directive_prefix(start_mark) + return (handle, prefix) + + def scan_tag_directive_handle(self, start_mark): + # See the specification for details. + value = self.scan_tag_handle('directive', start_mark) + ch = self.peek() + if ch != u' ': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch.encode('utf-8'), + self.get_mark()) + return value + + def scan_tag_directive_prefix(self, start_mark): + # See the specification for details. + value = self.scan_tag_uri('directive', start_mark) + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch.encode('utf-8'), + self.get_mark()) + return value + + def scan_directive_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + if self.peek() == u'#': + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in u'\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a comment or a line break, but found %r" + % ch.encode('utf-8'), self.get_mark()) + self.scan_line_break() + + def scan_anchor(self, TokenClass): + # The specification does not restrict characters for anchors and + # aliases. This may lead to problems, for instance, the document: + # [ *alias, value ] + # can be interpreted in two ways, as + # [ "value" ] + # and + # [ *alias , "value" ] + # Therefore we restrict aliases to numbers and ASCII letters. + start_mark = self.get_mark() + indicator = self.peek() + if indicator == u'*': + name = 'alias' + else: + name = 'anchor' + self.forward() + length = 0 + ch = self.peek(length) + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in u'\0 \t\r\n\x85\u2028\u2029?:,]}%@`': + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + end_mark = self.get_mark() + return TokenClass(value, start_mark, end_mark) + + def scan_tag(self): + # See the specification for details. + start_mark = self.get_mark() + ch = self.peek(1) + if ch == u'<': + handle = None + self.forward(2) + suffix = self.scan_tag_uri('tag', start_mark) + if self.peek() != u'>': + raise ScannerError("while parsing a tag", start_mark, + "expected '>', but found %r" % self.peek().encode('utf-8'), + self.get_mark()) + self.forward() + elif ch in u'\0 \t\r\n\x85\u2028\u2029': + handle = None + suffix = u'!' + self.forward() + else: + length = 1 + use_handle = False + while ch not in u'\0 \r\n\x85\u2028\u2029': + if ch == u'!': + use_handle = True + break + length += 1 + ch = self.peek(length) + handle = u'!' + if use_handle: + handle = self.scan_tag_handle('tag', start_mark) + else: + handle = u'!' + self.forward() + suffix = self.scan_tag_uri('tag', start_mark) + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a tag", start_mark, + "expected ' ', but found %r" % ch.encode('utf-8'), + self.get_mark()) + value = (handle, suffix) + end_mark = self.get_mark() + return TagToken(value, start_mark, end_mark) + + def scan_block_scalar(self, style): + # See the specification for details. + + if style == '>': + folded = True + else: + folded = False + + chunks = [] + start_mark = self.get_mark() + + # Scan the header. + self.forward() + chomping, increment = self.scan_block_scalar_indicators(start_mark) + self.scan_block_scalar_ignored_line(start_mark) + + # Determine the indentation level and go to the first non-empty line. + min_indent = self.indent+1 + if min_indent < 1: + min_indent = 1 + if increment is None: + breaks, max_indent, end_mark = self.scan_block_scalar_indentation() + indent = max(min_indent, max_indent) + else: + indent = min_indent+increment-1 + breaks, end_mark = self.scan_block_scalar_breaks(indent) + line_break = u'' + + # Scan the inner part of the block scalar. + while self.column == indent and self.peek() != u'\0': + chunks.extend(breaks) + leading_non_space = self.peek() not in u' \t' + length = 0 + while self.peek(length) not in u'\0\r\n\x85\u2028\u2029': + length += 1 + chunks.append(self.prefix(length)) + self.forward(length) + line_break = self.scan_line_break() + breaks, end_mark = self.scan_block_scalar_breaks(indent) + if self.column == indent and self.peek() != u'\0': + + # Unfortunately, folding rules are ambiguous. + # + # This is the folding according to the specification: + + if folded and line_break == u'\n' \ + and leading_non_space and self.peek() not in u' \t': + if not breaks: + chunks.append(u' ') + else: + chunks.append(line_break) + + # This is Clark Evans's interpretation (also in the spec + # examples): + # + #if folded and line_break == u'\n': + # if not breaks: + # if self.peek() not in ' \t': + # chunks.append(u' ') + # else: + # chunks.append(line_break) + #else: + # chunks.append(line_break) + else: + break + + # Chomp the tail. + if chomping is not False: + chunks.append(line_break) + if chomping is True: + chunks.extend(breaks) + + # We are done. + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) + + def scan_block_scalar_indicators(self, start_mark): + # See the specification for details. + chomping = None + increment = None + ch = self.peek() + if ch in u'+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch in u'0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + elif ch in u'0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + ch = self.peek() + if ch in u'+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected chomping or indentation indicators, but found %r" + % ch.encode('utf-8'), self.get_mark()) + return chomping, increment + + def scan_block_scalar_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + if self.peek() == u'#': + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in u'\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected a comment or a line break, but found %r" + % ch.encode('utf-8'), self.get_mark()) + self.scan_line_break() + + def scan_block_scalar_indentation(self): + # See the specification for details. + chunks = [] + max_indent = 0 + end_mark = self.get_mark() + while self.peek() in u' \r\n\x85\u2028\u2029': + if self.peek() != u' ': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + else: + self.forward() + if self.column > max_indent: + max_indent = self.column + return chunks, max_indent, end_mark + + def scan_block_scalar_breaks(self, indent): + # See the specification for details. + chunks = [] + end_mark = self.get_mark() + while self.column < indent and self.peek() == u' ': + self.forward() + while self.peek() in u'\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + while self.column < indent and self.peek() == u' ': + self.forward() + return chunks, end_mark + + def scan_flow_scalar(self, style): + # See the specification for details. + # Note that we loose indentation rules for quoted scalars. Quoted + # scalars don't need to adhere indentation because " and ' clearly + # mark the beginning and the end of them. Therefore we are less + # restrictive then the specification requires. We only need to check + # that document separators are not included in scalars. + if style == '"': + double = True + else: + double = False + chunks = [] + start_mark = self.get_mark() + quote = self.peek() + self.forward() + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + while self.peek() != quote: + chunks.extend(self.scan_flow_scalar_spaces(double, start_mark)) + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + self.forward() + end_mark = self.get_mark() + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) + + ESCAPE_REPLACEMENTS = { + u'0': u'\0', + u'a': u'\x07', + u'b': u'\x08', + u't': u'\x09', + u'\t': u'\x09', + u'n': u'\x0A', + u'v': u'\x0B', + u'f': u'\x0C', + u'r': u'\x0D', + u'e': u'\x1B', + u' ': u'\x20', + u'\"': u'\"', + u'\\': u'\\', + u'/': u'/', + u'N': u'\x85', + u'_': u'\xA0', + u'L': u'\u2028', + u'P': u'\u2029', + } + + ESCAPE_CODES = { + u'x': 2, + u'u': 4, + u'U': 8, + } + + def scan_flow_scalar_non_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + length = 0 + while self.peek(length) not in u'\'\"\\\0 \t\r\n\x85\u2028\u2029': + length += 1 + if length: + chunks.append(self.prefix(length)) + self.forward(length) + ch = self.peek() + if not double and ch == u'\'' and self.peek(1) == u'\'': + chunks.append(u'\'') + self.forward(2) + elif (double and ch == u'\'') or (not double and ch in u'\"\\'): + chunks.append(ch) + self.forward() + elif double and ch == u'\\': + self.forward() + ch = self.peek() + if ch in self.ESCAPE_REPLACEMENTS: + chunks.append(self.ESCAPE_REPLACEMENTS[ch]) + self.forward() + elif ch in self.ESCAPE_CODES: + length = self.ESCAPE_CODES[ch] + self.forward() + for k in range(length): + if self.peek(k) not in u'0123456789ABCDEFabcdef': + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "expected escape sequence of %d hexdecimal numbers, but found %r" % + (length, self.peek(k).encode('utf-8')), self.get_mark()) + code = int(self.prefix(length), 16) + chunks.append(unichr(code)) + self.forward(length) + elif ch in u'\r\n\x85\u2028\u2029': + self.scan_line_break() + chunks.extend(self.scan_flow_scalar_breaks(double, start_mark)) + else: + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "found unknown escape character %r" % ch.encode('utf-8'), self.get_mark()) + else: + return chunks + + def scan_flow_scalar_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + length = 0 + while self.peek(length) in u' \t': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch == u'\0': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected end of stream", self.get_mark()) + elif ch in u'\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + breaks = self.scan_flow_scalar_breaks(double, start_mark) + if line_break != u'\n': + chunks.append(line_break) + elif not breaks: + chunks.append(u' ') + chunks.extend(breaks) + else: + chunks.append(whitespaces) + return chunks + + def scan_flow_scalar_breaks(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + # Instead of checking indentation, we check for document + # separators. + prefix = self.prefix(3) + if (prefix == u'---' or prefix == u'...') \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected document separator", self.get_mark()) + while self.peek() in u' \t': + self.forward() + if self.peek() in u'\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + else: + return chunks + + def scan_plain(self): + # See the specification for details. + # We add an additional restriction for the flow context: + # plain scalars in the flow context cannot contain ',' or '?'. + # We also keep track of the `allow_simple_key` flag here. + # Indentation rules are loosed for the flow context. + chunks = [] + start_mark = self.get_mark() + end_mark = start_mark + indent = self.indent+1 + # We allow zero indentation for scalars, but then we need to check for + # document separators at the beginning of the line. + #if indent == 0: + # indent = 1 + spaces = [] + while True: + length = 0 + if self.peek() == u'#': + break + while True: + ch = self.peek(length) + if ch in u'\0 \t\r\n\x85\u2028\u2029' \ + or (ch == u':' and + self.peek(length+1) in u'\0 \t\r\n\x85\u2028\u2029' + + (u',[]{}' if self.flow_level else u''))\ + or (self.flow_level and ch in u',?[]{}'): + break + length += 1 + if length == 0: + break + self.allow_simple_key = False + chunks.extend(spaces) + chunks.append(self.prefix(length)) + self.forward(length) + end_mark = self.get_mark() + spaces = self.scan_plain_spaces(indent, start_mark) + if not spaces or self.peek() == u'#' \ + or (not self.flow_level and self.column < indent): + break + return ScalarToken(u''.join(chunks), True, start_mark, end_mark) + + def scan_plain_spaces(self, indent, start_mark): + # See the specification for details. + # The specification is really confusing about tabs in plain scalars. + # We just forbid them completely. Do not use tabs in YAML! + chunks = [] + length = 0 + while self.peek(length) in u' ': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch in u'\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + self.allow_simple_key = True + prefix = self.prefix(3) + if (prefix == u'---' or prefix == u'...') \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return + breaks = [] + while self.peek() in u' \r\n\x85\u2028\u2029': + if self.peek() == ' ': + self.forward() + else: + breaks.append(self.scan_line_break()) + prefix = self.prefix(3) + if (prefix == u'---' or prefix == u'...') \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return + if line_break != u'\n': + chunks.append(line_break) + elif not breaks: + chunks.append(u' ') + chunks.extend(breaks) + elif whitespaces: + chunks.append(whitespaces) + return chunks + + def scan_tag_handle(self, name, start_mark): + # See the specification for details. + # For some strange reasons, the specification does not allow '_' in + # tag handles. I have allowed it anyway. + ch = self.peek() + if ch != u'!': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch.encode('utf-8'), + self.get_mark()) + length = 1 + ch = self.peek(length) + if ch != u' ': + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-_': + length += 1 + ch = self.peek(length) + if ch != u'!': + self.forward(length) + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch.encode('utf-8'), + self.get_mark()) + length += 1 + value = self.prefix(length) + self.forward(length) + return value + + def scan_tag_uri(self, name, start_mark): + # See the specification for details. + # Note: we do not check if URI is well-formed. + chunks = [] + length = 0 + ch = self.peek(length) + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-;/?:@&=+$,_.!~*\'()[]%': + if ch == u'%': + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + chunks.append(self.scan_uri_escapes(name, start_mark)) + else: + length += 1 + ch = self.peek(length) + if length: + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + if not chunks: + raise ScannerError("while parsing a %s" % name, start_mark, + "expected URI, but found %r" % ch.encode('utf-8'), + self.get_mark()) + return u''.join(chunks) + + def scan_uri_escapes(self, name, start_mark): + # See the specification for details. + bytes = [] + mark = self.get_mark() + while self.peek() == u'%': + self.forward() + for k in range(2): + if self.peek(k) not in u'0123456789ABCDEFabcdef': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected URI escape sequence of 2 hexdecimal numbers, but found %r" % + (self.peek(k).encode('utf-8')), self.get_mark()) + bytes.append(chr(int(self.prefix(2), 16))) + self.forward(2) + try: + value = unicode(''.join(bytes), 'utf-8') + except UnicodeDecodeError, exc: + raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark) + return value + + def scan_line_break(self): + # Transforms: + # '\r\n' : '\n' + # '\r' : '\n' + # '\n' : '\n' + # '\x85' : '\n' + # '\u2028' : '\u2028' + # '\u2029 : '\u2029' + # default : '' + ch = self.peek() + if ch in u'\r\n\x85': + if self.prefix(2) == u'\r\n': + self.forward(2) + else: + self.forward() + return u'\n' + elif ch in u'\u2028\u2029': + self.forward() + return ch + return u'' diff --git a/pipenv/patched/yaml2/serializer.py b/pipenv/patched/yaml2/serializer.py new file mode 100644 index 00000000..0bf1e96d --- /dev/null +++ b/pipenv/patched/yaml2/serializer.py @@ -0,0 +1,111 @@ + +__all__ = ['Serializer', 'SerializerError'] + +from error import YAMLError +from events import * +from nodes import * + +class SerializerError(YAMLError): + pass + +class Serializer(object): + + ANCHOR_TEMPLATE = u'id%03d' + + def __init__(self, encoding=None, + explicit_start=None, explicit_end=None, version=None, tags=None): + self.use_encoding = encoding + self.use_explicit_start = explicit_start + self.use_explicit_end = explicit_end + self.use_version = version + self.use_tags = tags + self.serialized_nodes = {} + self.anchors = {} + self.last_anchor_id = 0 + self.closed = None + + def open(self): + if self.closed is None: + self.emit(StreamStartEvent(encoding=self.use_encoding)) + self.closed = False + elif self.closed: + raise SerializerError("serializer is closed") + else: + raise SerializerError("serializer is already opened") + + def close(self): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif not self.closed: + self.emit(StreamEndEvent()) + self.closed = True + + #def __del__(self): + # self.close() + + def serialize(self, node): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif self.closed: + raise SerializerError("serializer is closed") + self.emit(DocumentStartEvent(explicit=self.use_explicit_start, + version=self.use_version, tags=self.use_tags)) + self.anchor_node(node) + self.serialize_node(node, None, None) + self.emit(DocumentEndEvent(explicit=self.use_explicit_end)) + self.serialized_nodes = {} + self.anchors = {} + self.last_anchor_id = 0 + + def anchor_node(self, node): + if node in self.anchors: + if self.anchors[node] is None: + self.anchors[node] = self.generate_anchor(node) + else: + self.anchors[node] = None + if isinstance(node, SequenceNode): + for item in node.value: + self.anchor_node(item) + elif isinstance(node, MappingNode): + for key, value in node.value: + self.anchor_node(key) + self.anchor_node(value) + + def generate_anchor(self, node): + self.last_anchor_id += 1 + return self.ANCHOR_TEMPLATE % self.last_anchor_id + + def serialize_node(self, node, parent, index): + alias = self.anchors[node] + if node in self.serialized_nodes: + self.emit(AliasEvent(alias)) + else: + self.serialized_nodes[node] = True + self.descend_resolver(parent, index) + if isinstance(node, ScalarNode): + detected_tag = self.resolve(ScalarNode, node.value, (True, False)) + default_tag = self.resolve(ScalarNode, node.value, (False, True)) + implicit = (node.tag == detected_tag), (node.tag == default_tag) + self.emit(ScalarEvent(alias, node.tag, implicit, node.value, + style=node.style)) + elif isinstance(node, SequenceNode): + implicit = (node.tag + == self.resolve(SequenceNode, node.value, True)) + self.emit(SequenceStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + index = 0 + for item in node.value: + self.serialize_node(item, node, index) + index += 1 + self.emit(SequenceEndEvent()) + elif isinstance(node, MappingNode): + implicit = (node.tag + == self.resolve(MappingNode, node.value, True)) + self.emit(MappingStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + for key, value in node.value: + self.serialize_node(key, node, None) + self.serialize_node(value, node, key) + self.emit(MappingEndEvent()) + self.ascend_resolver() + diff --git a/pipenv/patched/yaml2/tokens.py b/pipenv/patched/yaml2/tokens.py new file mode 100644 index 00000000..4d0b48a3 --- /dev/null +++ b/pipenv/patched/yaml2/tokens.py @@ -0,0 +1,104 @@ + +class Token(object): + def __init__(self, start_mark, end_mark): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in self.__dict__ + if not key.endswith('_mark')] + attributes.sort() + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +#class BOMToken(Token): +# id = '' + +class DirectiveToken(Token): + id = '' + def __init__(self, name, value, start_mark, end_mark): + self.name = name + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class DocumentStartToken(Token): + id = '' + +class DocumentEndToken(Token): + id = '' + +class StreamStartToken(Token): + id = '' + def __init__(self, start_mark=None, end_mark=None, + encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndToken(Token): + id = '' + +class BlockSequenceStartToken(Token): + id = '' + +class BlockMappingStartToken(Token): + id = '' + +class BlockEndToken(Token): + id = '' + +class FlowSequenceStartToken(Token): + id = '[' + +class FlowMappingStartToken(Token): + id = '{' + +class FlowSequenceEndToken(Token): + id = ']' + +class FlowMappingEndToken(Token): + id = '}' + +class KeyToken(Token): + id = '?' + +class ValueToken(Token): + id = ':' + +class BlockEntryToken(Token): + id = '-' + +class FlowEntryToken(Token): + id = ',' + +class AliasToken(Token): + id = '' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class AnchorToken(Token): + id = '' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class TagToken(Token): + id = '' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class ScalarToken(Token): + id = '' + def __init__(self, value, plain, start_mark, end_mark, style=None): + self.value = value + self.plain = plain + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + diff --git a/pipenv/patched/yaml3/LICENSE b/pipenv/patched/yaml3/LICENSE new file mode 100644 index 00000000..3d82c281 --- /dev/null +++ b/pipenv/patched/yaml3/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2017-2020 Ingy döt Net +Copyright (c) 2006-2016 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/patched/yaml3/__init__.py b/pipenv/patched/yaml3/__init__.py new file mode 100644 index 00000000..13d687c5 --- /dev/null +++ b/pipenv/patched/yaml3/__init__.py @@ -0,0 +1,427 @@ + +from .error import * + +from .tokens import * +from .events import * +from .nodes import * + +from .loader import * +from .dumper import * + +__version__ = '5.3.1' +try: + from .cyaml import * + __with_libyaml__ = True +except ImportError: + __with_libyaml__ = False + +import io + +#------------------------------------------------------------------------------ +# Warnings control +#------------------------------------------------------------------------------ + +# 'Global' warnings state: +_warnings_enabled = { + 'YAMLLoadWarning': True, +} + +# Get or set global warnings' state +def warnings(settings=None): + if settings is None: + return _warnings_enabled + + if type(settings) is dict: + for key in settings: + if key in _warnings_enabled: + _warnings_enabled[key] = settings[key] + +# Warn when load() is called without Loader=... +class YAMLLoadWarning(RuntimeWarning): + pass + +def load_warning(method): + if _warnings_enabled['YAMLLoadWarning'] is False: + return + + import warnings + + message = ( + "calling yaml.%s() without Loader=... is deprecated, as the " + "default Loader is unsafe. Please read " + "https://msg.pyyaml.org/load for full details." + ) % method + + warnings.warn(message, YAMLLoadWarning, stacklevel=3) + +#------------------------------------------------------------------------------ +def scan(stream, Loader=Loader): + """ + Scan a YAML stream and produce scanning tokens. + """ + loader = Loader(stream) + try: + while loader.check_token(): + yield loader.get_token() + finally: + loader.dispose() + +def parse(stream, Loader=Loader): + """ + Parse a YAML stream and produce parsing events. + """ + loader = Loader(stream) + try: + while loader.check_event(): + yield loader.get_event() + finally: + loader.dispose() + +def compose(stream, Loader=Loader): + """ + Parse the first YAML document in a stream + and produce the corresponding representation tree. + """ + loader = Loader(stream) + try: + return loader.get_single_node() + finally: + loader.dispose() + +def compose_all(stream, Loader=Loader): + """ + Parse all YAML documents in a stream + and produce corresponding representation trees. + """ + loader = Loader(stream) + try: + while loader.check_node(): + yield loader.get_node() + finally: + loader.dispose() + +def load(stream, Loader=None): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + """ + if Loader is None: + load_warning('load') + Loader = FullLoader + + loader = Loader(stream) + try: + return loader.get_single_data() + finally: + loader.dispose() + +def load_all(stream, Loader=None): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + """ + if Loader is None: + load_warning('load_all') + Loader = FullLoader + + loader = Loader(stream) + try: + while loader.check_data(): + yield loader.get_data() + finally: + loader.dispose() + +def full_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load(stream, FullLoader) + +def full_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load_all(stream, FullLoader) + +def safe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. + """ + return load(stream, SafeLoader) + +def safe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. + """ + return load_all(stream, SafeLoader) + +def unsafe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load(stream, UnsafeLoader) + +def unsafe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load_all(stream, UnsafeLoader) + +def emit(events, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + """ + Emit YAML parsing events into a stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + stream = io.StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + try: + for event in events: + dumper.emit(event) + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def serialize_all(nodes, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + """ + Serialize a sequence of representation trees into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + if encoding is None: + stream = io.StringIO() + else: + stream = io.BytesIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end) + try: + dumper.open() + for node in nodes: + dumper.serialize(node) + dumper.close() + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def serialize(node, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a representation tree into a YAML stream. + If stream is None, return the produced string instead. + """ + return serialize_all([node], stream, Dumper=Dumper, **kwds) + +def dump_all(documents, stream=None, Dumper=Dumper, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + """ + Serialize a sequence of Python objects into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + if encoding is None: + stream = io.StringIO() + else: + stream = io.BytesIO() + getvalue = stream.getvalue + dumper = Dumper(stream, default_style=default_style, + default_flow_style=default_flow_style, + canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys) + try: + dumper.open() + for data in documents: + dumper.represent(data) + dumper.close() + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def dump(data, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a Python object into a YAML stream. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=Dumper, **kwds) + +def safe_dump_all(documents, stream=None, **kwds): + """ + Serialize a sequence of Python objects into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all(documents, stream, Dumper=SafeDumper, **kwds) + +def safe_dump(data, stream=None, **kwds): + """ + Serialize a Python object into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=SafeDumper, **kwds) + +def add_implicit_resolver(tag, regexp, first=None, + Loader=None, Dumper=Dumper): + """ + Add an implicit scalar detector. + If an implicit scalar value matches the given regexp, + the corresponding tag is assigned to the scalar. + first is a sequence of possible initial characters or None. + """ + if Loader is None: + loader.Loader.add_implicit_resolver(tag, regexp, first) + loader.FullLoader.add_implicit_resolver(tag, regexp, first) + loader.UnsafeLoader.add_implicit_resolver(tag, regexp, first) + else: + Loader.add_implicit_resolver(tag, regexp, first) + Dumper.add_implicit_resolver(tag, regexp, first) + +def add_path_resolver(tag, path, kind=None, Loader=None, Dumper=Dumper): + """ + Add a path based resolver for the given tag. + A path is a list of keys that forms a path + to a node in the representation tree. + Keys can be string values, integers, or None. + """ + if Loader is None: + loader.Loader.add_path_resolver(tag, path, kind) + loader.FullLoader.add_path_resolver(tag, path, kind) + loader.UnsafeLoader.add_path_resolver(tag, path, kind) + else: + Loader.add_path_resolver(tag, path, kind) + Dumper.add_path_resolver(tag, path, kind) + +def add_constructor(tag, constructor, Loader=None): + """ + Add a constructor for the given tag. + Constructor is a function that accepts a Loader instance + and a node object and produces the corresponding Python object. + """ + if Loader is None: + loader.Loader.add_constructor(tag, constructor) + loader.FullLoader.add_constructor(tag, constructor) + loader.UnsafeLoader.add_constructor(tag, constructor) + else: + Loader.add_constructor(tag, constructor) + +def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): + """ + Add a multi-constructor for the given tag prefix. + Multi-constructor is called for a node if its tag starts with tag_prefix. + Multi-constructor accepts a Loader instance, a tag suffix, + and a node object and produces the corresponding Python object. + """ + if Loader is None: + loader.Loader.add_multi_constructor(tag_prefix, multi_constructor) + loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor) + loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor) + else: + Loader.add_multi_constructor(tag_prefix, multi_constructor) + +def add_representer(data_type, representer, Dumper=Dumper): + """ + Add a representer for the given type. + Representer is a function accepting a Dumper instance + and an instance of the given data type + and producing the corresponding representation node. + """ + Dumper.add_representer(data_type, representer) + +def add_multi_representer(data_type, multi_representer, Dumper=Dumper): + """ + Add a representer for the given type. + Multi-representer is a function accepting a Dumper instance + and an instance of the given data type or subtype + and producing the corresponding representation node. + """ + Dumper.add_multi_representer(data_type, multi_representer) + +class YAMLObjectMetaclass(type): + """ + The metaclass for YAMLObject. + """ + def __init__(cls, name, bases, kwds): + super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds) + if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None: + if isinstance(cls.yaml_loader, list): + for loader in cls.yaml_loader: + loader.add_constructor(cls.yaml_tag, cls.from_yaml) + else: + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + + cls.yaml_dumper.add_representer(cls, cls.to_yaml) + +class YAMLObject(metaclass=YAMLObjectMetaclass): + """ + An object that can dump itself to a YAML stream + and load itself from a YAML stream. + """ + + __slots__ = () # no direct instantiation, so allow immutable subclasses + + yaml_loader = [Loader, FullLoader, UnsafeLoader] + yaml_dumper = Dumper + + yaml_tag = None + yaml_flow_style = None + + @classmethod + def from_yaml(cls, loader, node): + """ + Convert a representation node to a Python object. + """ + return loader.construct_yaml_object(node, cls) + + @classmethod + def to_yaml(cls, dumper, data): + """ + Convert a Python object to a representation node. + """ + return dumper.represent_yaml_object(cls.yaml_tag, data, cls, + flow_style=cls.yaml_flow_style) + diff --git a/pipenv/patched/yaml3/composer.py b/pipenv/patched/yaml3/composer.py new file mode 100644 index 00000000..6d15cb40 --- /dev/null +++ b/pipenv/patched/yaml3/composer.py @@ -0,0 +1,139 @@ + +__all__ = ['Composer', 'ComposerError'] + +from .error import MarkedYAMLError +from .events import * +from .nodes import * + +class ComposerError(MarkedYAMLError): + pass + +class Composer: + + def __init__(self): + self.anchors = {} + + def check_node(self): + # Drop the STREAM-START event. + if self.check_event(StreamStartEvent): + self.get_event() + + # If there are more documents available? + return not self.check_event(StreamEndEvent) + + def get_node(self): + # Get the root node of the next document. + if not self.check_event(StreamEndEvent): + return self.compose_document() + + def get_single_node(self): + # Drop the STREAM-START event. + self.get_event() + + # Compose a document if the stream is not empty. + document = None + if not self.check_event(StreamEndEvent): + document = self.compose_document() + + # Ensure that the stream contains no more documents. + if not self.check_event(StreamEndEvent): + event = self.get_event() + raise ComposerError("expected a single document in the stream", + document.start_mark, "but found another document", + event.start_mark) + + # Drop the STREAM-END event. + self.get_event() + + return document + + def compose_document(self): + # Drop the DOCUMENT-START event. + self.get_event() + + # Compose the root node. + node = self.compose_node(None, None) + + # Drop the DOCUMENT-END event. + self.get_event() + + self.anchors = {} + return node + + def compose_node(self, parent, index): + if self.check_event(AliasEvent): + event = self.get_event() + anchor = event.anchor + if anchor not in self.anchors: + raise ComposerError(None, None, "found undefined alias %r" + % anchor, event.start_mark) + return self.anchors[anchor] + event = self.peek_event() + anchor = event.anchor + if anchor is not None: + if anchor in self.anchors: + raise ComposerError("found duplicate anchor %r; first occurrence" + % anchor, self.anchors[anchor].start_mark, + "second occurrence", event.start_mark) + self.descend_resolver(parent, index) + if self.check_event(ScalarEvent): + node = self.compose_scalar_node(anchor) + elif self.check_event(SequenceStartEvent): + node = self.compose_sequence_node(anchor) + elif self.check_event(MappingStartEvent): + node = self.compose_mapping_node(anchor) + self.ascend_resolver() + return node + + def compose_scalar_node(self, anchor): + event = self.get_event() + tag = event.tag + if tag is None or tag == '!': + tag = self.resolve(ScalarNode, event.value, event.implicit) + node = ScalarNode(tag, event.value, + event.start_mark, event.end_mark, style=event.style) + if anchor is not None: + self.anchors[anchor] = node + return node + + def compose_sequence_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == '!': + tag = self.resolve(SequenceNode, None, start_event.implicit) + node = SequenceNode(tag, [], + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + index = 0 + while not self.check_event(SequenceEndEvent): + node.value.append(self.compose_node(node, index)) + index += 1 + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + + def compose_mapping_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == '!': + tag = self.resolve(MappingNode, None, start_event.implicit) + node = MappingNode(tag, [], + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + while not self.check_event(MappingEndEvent): + #key_event = self.peek_event() + item_key = self.compose_node(node, None) + #if item_key in node.value: + # raise ComposerError("while composing a mapping", start_event.start_mark, + # "found duplicate key", key_event.start_mark) + item_value = self.compose_node(node, item_key) + #node.value[item_key] = item_value + node.value.append((item_key, item_value)) + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + diff --git a/pipenv/patched/yaml3/constructor.py b/pipenv/patched/yaml3/constructor.py new file mode 100644 index 00000000..1948b125 --- /dev/null +++ b/pipenv/patched/yaml3/constructor.py @@ -0,0 +1,748 @@ + +__all__ = [ + 'BaseConstructor', + 'SafeConstructor', + 'FullConstructor', + 'UnsafeConstructor', + 'Constructor', + 'ConstructorError' +] + +from .error import * +from .nodes import * + +import collections.abc, datetime, base64, binascii, re, sys, types + +class ConstructorError(MarkedYAMLError): + pass + +class BaseConstructor: + + yaml_constructors = {} + yaml_multi_constructors = {} + + def __init__(self): + self.constructed_objects = {} + self.recursive_objects = {} + self.state_generators = [] + self.deep_construct = False + + def check_data(self): + # If there are more documents available? + return self.check_node() + + def check_state_key(self, key): + """Block special attributes/methods from being set in a newly created + object, to prevent user-controlled methods from being called during + deserialization""" + if self.get_state_keys_blacklist_regexp().match(key): + raise ConstructorError(None, None, + "blacklisted key '%s' in instance state found" % (key,), None) + + def get_data(self): + # Construct and return the next document. + if self.check_node(): + return self.construct_document(self.get_node()) + + def get_single_data(self): + # Ensure that the stream contains a single document and construct it. + node = self.get_single_node() + if node is not None: + return self.construct_document(node) + return None + + def construct_document(self, node): + data = self.construct_object(node) + while self.state_generators: + state_generators = self.state_generators + self.state_generators = [] + for generator in state_generators: + for dummy in generator: + pass + self.constructed_objects = {} + self.recursive_objects = {} + self.deep_construct = False + return data + + def construct_object(self, node, deep=False): + if node in self.constructed_objects: + return self.constructed_objects[node] + if deep: + old_deep = self.deep_construct + self.deep_construct = True + if node in self.recursive_objects: + raise ConstructorError(None, None, + "found unconstructable recursive node", node.start_mark) + self.recursive_objects[node] = None + constructor = None + tag_suffix = None + if node.tag in self.yaml_constructors: + constructor = self.yaml_constructors[node.tag] + else: + for tag_prefix in self.yaml_multi_constructors: + if tag_prefix is not None and node.tag.startswith(tag_prefix): + tag_suffix = node.tag[len(tag_prefix):] + constructor = self.yaml_multi_constructors[tag_prefix] + break + else: + if None in self.yaml_multi_constructors: + tag_suffix = node.tag + constructor = self.yaml_multi_constructors[None] + elif None in self.yaml_constructors: + constructor = self.yaml_constructors[None] + elif isinstance(node, ScalarNode): + constructor = self.__class__.construct_scalar + elif isinstance(node, SequenceNode): + constructor = self.__class__.construct_sequence + elif isinstance(node, MappingNode): + constructor = self.__class__.construct_mapping + if tag_suffix is None: + data = constructor(self, node) + else: + data = constructor(self, tag_suffix, node) + if isinstance(data, types.GeneratorType): + generator = data + data = next(generator) + if self.deep_construct: + for dummy in generator: + pass + else: + self.state_generators.append(generator) + self.constructed_objects[node] = data + del self.recursive_objects[node] + if deep: + self.deep_construct = old_deep + return data + + def construct_scalar(self, node): + if not isinstance(node, ScalarNode): + raise ConstructorError(None, None, + "expected a scalar node, but found %s" % node.id, + node.start_mark) + return node.value + + def construct_sequence(self, node, deep=False): + if not isinstance(node, SequenceNode): + raise ConstructorError(None, None, + "expected a sequence node, but found %s" % node.id, + node.start_mark) + return [self.construct_object(child, deep=deep) + for child in node.value] + + def construct_mapping(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + mapping = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if not isinstance(key, collections.abc.Hashable): + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unhashable key", key_node.start_mark) + value = self.construct_object(value_node, deep=deep) + mapping[key] = value + return mapping + + def construct_pairs(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + pairs = [] + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + value = self.construct_object(value_node, deep=deep) + pairs.append((key, value)) + return pairs + + @classmethod + def add_constructor(cls, tag, constructor): + if not 'yaml_constructors' in cls.__dict__: + cls.yaml_constructors = cls.yaml_constructors.copy() + cls.yaml_constructors[tag] = constructor + + @classmethod + def add_multi_constructor(cls, tag_prefix, multi_constructor): + if not 'yaml_multi_constructors' in cls.__dict__: + cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() + cls.yaml_multi_constructors[tag_prefix] = multi_constructor + +class SafeConstructor(BaseConstructor): + + def construct_scalar(self, node): + if isinstance(node, MappingNode): + for key_node, value_node in node.value: + if key_node.tag == 'tag:yaml.org,2002:value': + return self.construct_scalar(value_node) + return super().construct_scalar(node) + + def flatten_mapping(self, node): + merge = [] + index = 0 + while index < len(node.value): + key_node, value_node = node.value[index] + if key_node.tag == 'tag:yaml.org,2002:merge': + del node.value[index] + if isinstance(value_node, MappingNode): + self.flatten_mapping(value_node) + merge.extend(value_node.value) + elif isinstance(value_node, SequenceNode): + submerge = [] + for subnode in value_node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing a mapping", + node.start_mark, + "expected a mapping for merging, but found %s" + % subnode.id, subnode.start_mark) + self.flatten_mapping(subnode) + submerge.append(subnode.value) + submerge.reverse() + for value in submerge: + merge.extend(value) + else: + raise ConstructorError("while constructing a mapping", node.start_mark, + "expected a mapping or list of mappings for merging, but found %s" + % value_node.id, value_node.start_mark) + elif key_node.tag == 'tag:yaml.org,2002:value': + key_node.tag = 'tag:yaml.org,2002:str' + index += 1 + else: + index += 1 + if merge: + node.value = merge + node.value + + def construct_mapping(self, node, deep=False): + if isinstance(node, MappingNode): + self.flatten_mapping(node) + return super().construct_mapping(node, deep=deep) + + def construct_yaml_null(self, node): + self.construct_scalar(node) + return None + + bool_values = { + 'yes': True, + 'no': False, + 'true': True, + 'false': False, + 'on': True, + 'off': False, + } + + def construct_yaml_bool(self, node): + value = self.construct_scalar(node) + return self.bool_values[value.lower()] + + def construct_yaml_int(self, node): + value = self.construct_scalar(node) + value = value.replace('_', '') + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '0': + return 0 + elif value.startswith('0b'): + return sign*int(value[2:], 2) + elif value.startswith('0x'): + return sign*int(value[2:], 16) + elif value[0] == '0': + return sign*int(value, 8) + elif ':' in value: + digits = [int(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*int(value) + + inf_value = 1e300 + while inf_value != inf_value*inf_value: + inf_value *= inf_value + nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99). + + def construct_yaml_float(self, node): + value = self.construct_scalar(node) + value = value.replace('_', '').lower() + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '.inf': + return sign*self.inf_value + elif value == '.nan': + return self.nan_value + elif ':' in value: + digits = [float(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0.0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*float(value) + + def construct_yaml_binary(self, node): + try: + value = self.construct_scalar(node).encode('ascii') + except UnicodeEncodeError as exc: + raise ConstructorError(None, None, + "failed to convert base64 data into ascii: %s" % exc, + node.start_mark) + try: + if hasattr(base64, 'decodebytes'): + return base64.decodebytes(value) + else: + return base64.decodestring(value) + except binascii.Error as exc: + raise ConstructorError(None, None, + "failed to decode base64 data: %s" % exc, node.start_mark) + + timestamp_regexp = re.compile( + r'''^(?P[0-9][0-9][0-9][0-9]) + -(?P[0-9][0-9]?) + -(?P[0-9][0-9]?) + (?:(?:[Tt]|[ \t]+) + (?P[0-9][0-9]?) + :(?P[0-9][0-9]) + :(?P[0-9][0-9]) + (?:\.(?P[0-9]*))? + (?:[ \t]*(?PZ|(?P[-+])(?P[0-9][0-9]?) + (?::(?P[0-9][0-9]))?))?)?$''', re.X) + + def construct_yaml_timestamp(self, node): + value = self.construct_scalar(node) + match = self.timestamp_regexp.match(node.value) + values = match.groupdict() + year = int(values['year']) + month = int(values['month']) + day = int(values['day']) + if not values['hour']: + return datetime.date(year, month, day) + hour = int(values['hour']) + minute = int(values['minute']) + second = int(values['second']) + fraction = 0 + tzinfo = None + if values['fraction']: + fraction = values['fraction'][:6] + while len(fraction) < 6: + fraction += '0' + fraction = int(fraction) + if values['tz_sign']: + tz_hour = int(values['tz_hour']) + tz_minute = int(values['tz_minute'] or 0) + delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute) + if values['tz_sign'] == '-': + delta = -delta + tzinfo = datetime.timezone(delta) + elif values['tz']: + tzinfo = datetime.timezone.utc + return datetime.datetime(year, month, day, hour, minute, second, fraction, + tzinfo=tzinfo) + + def construct_yaml_omap(self, node): + # Note: we do not check for duplicate keys, because it's too + # CPU-expensive. + omap = [] + yield omap + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node, value_node = subnode.value[0] + key = self.construct_object(key_node) + value = self.construct_object(value_node) + omap.append((key, value)) + + def construct_yaml_pairs(self, node): + # Note: the same code as `construct_yaml_omap`. + pairs = [] + yield pairs + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node, value_node = subnode.value[0] + key = self.construct_object(key_node) + value = self.construct_object(value_node) + pairs.append((key, value)) + + def construct_yaml_set(self, node): + data = set() + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_str(self, node): + return self.construct_scalar(node) + + def construct_yaml_seq(self, node): + data = [] + yield data + data.extend(self.construct_sequence(node)) + + def construct_yaml_map(self, node): + data = {} + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_object(self, node, cls): + data = cls.__new__(cls) + yield data + if hasattr(data, '__setstate__'): + state = self.construct_mapping(node, deep=True) + data.__setstate__(state) + else: + state = self.construct_mapping(node) + data.__dict__.update(state) + + def construct_undefined(self, node): + raise ConstructorError(None, None, + "could not determine a constructor for the tag %r" % node.tag, + node.start_mark) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:null', + SafeConstructor.construct_yaml_null) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:bool', + SafeConstructor.construct_yaml_bool) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:int', + SafeConstructor.construct_yaml_int) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:float', + SafeConstructor.construct_yaml_float) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:binary', + SafeConstructor.construct_yaml_binary) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:timestamp', + SafeConstructor.construct_yaml_timestamp) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:omap', + SafeConstructor.construct_yaml_omap) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:pairs', + SafeConstructor.construct_yaml_pairs) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:set', + SafeConstructor.construct_yaml_set) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:str', + SafeConstructor.construct_yaml_str) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:seq', + SafeConstructor.construct_yaml_seq) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:map', + SafeConstructor.construct_yaml_map) + +SafeConstructor.add_constructor(None, + SafeConstructor.construct_undefined) + +class FullConstructor(SafeConstructor): + # 'extend' is blacklisted because it is used by + # construct_python_object_apply to add `listitems` to a newly generate + # python instance + def get_state_keys_blacklist(self): + return ['^extend$', '^__.*__$'] + + def get_state_keys_blacklist_regexp(self): + if not hasattr(self, 'state_keys_blacklist_regexp'): + self.state_keys_blacklist_regexp = re.compile('(' + '|'.join(self.get_state_keys_blacklist()) + ')') + return self.state_keys_blacklist_regexp + + def construct_python_str(self, node): + return self.construct_scalar(node) + + def construct_python_unicode(self, node): + return self.construct_scalar(node) + + def construct_python_bytes(self, node): + try: + value = self.construct_scalar(node).encode('ascii') + except UnicodeEncodeError as exc: + raise ConstructorError(None, None, + "failed to convert base64 data into ascii: %s" % exc, + node.start_mark) + try: + if hasattr(base64, 'decodebytes'): + return base64.decodebytes(value) + else: + return base64.decodestring(value) + except binascii.Error as exc: + raise ConstructorError(None, None, + "failed to decode base64 data: %s" % exc, node.start_mark) + + def construct_python_long(self, node): + return self.construct_yaml_int(node) + + def construct_python_complex(self, node): + return complex(self.construct_scalar(node)) + + def construct_python_tuple(self, node): + return tuple(self.construct_sequence(node)) + + def find_python_module(self, name, mark, unsafe=False): + if not name: + raise ConstructorError("while constructing a Python module", mark, + "expected non-empty name appended to the tag", mark) + if unsafe: + try: + __import__(name) + except ImportError as exc: + raise ConstructorError("while constructing a Python module", mark, + "cannot find module %r (%s)" % (name, exc), mark) + if name not in sys.modules: + raise ConstructorError("while constructing a Python module", mark, + "module %r is not imported" % name, mark) + return sys.modules[name] + + def find_python_name(self, name, mark, unsafe=False): + if not name: + raise ConstructorError("while constructing a Python object", mark, + "expected non-empty name appended to the tag", mark) + if '.' in name: + module_name, object_name = name.rsplit('.', 1) + else: + module_name = 'builtins' + object_name = name + if unsafe: + try: + __import__(module_name) + except ImportError as exc: + raise ConstructorError("while constructing a Python object", mark, + "cannot find module %r (%s)" % (module_name, exc), mark) + if module_name not in sys.modules: + raise ConstructorError("while constructing a Python object", mark, + "module %r is not imported" % module_name, mark) + module = sys.modules[module_name] + if not hasattr(module, object_name): + raise ConstructorError("while constructing a Python object", mark, + "cannot find %r in the module %r" + % (object_name, module.__name__), mark) + return getattr(module, object_name) + + def construct_python_name(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python name", node.start_mark, + "expected the empty value, but found %r" % value, node.start_mark) + return self.find_python_name(suffix, node.start_mark) + + def construct_python_module(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python module", node.start_mark, + "expected the empty value, but found %r" % value, node.start_mark) + return self.find_python_module(suffix, node.start_mark) + + def make_python_instance(self, suffix, node, + args=None, kwds=None, newobj=False, unsafe=False): + if not args: + args = [] + if not kwds: + kwds = {} + cls = self.find_python_name(suffix, node.start_mark) + if not (unsafe or isinstance(cls, type)): + raise ConstructorError("while constructing a Python instance", node.start_mark, + "expected a class, but found %r" % type(cls), + node.start_mark) + if newobj and isinstance(cls, type): + return cls.__new__(cls, *args, **kwds) + else: + return cls(*args, **kwds) + + def set_python_instance_state(self, instance, state, unsafe=False): + if hasattr(instance, '__setstate__'): + instance.__setstate__(state) + else: + slotstate = {} + if isinstance(state, tuple) and len(state) == 2: + state, slotstate = state + if hasattr(instance, '__dict__'): + if not unsafe and state: + for key in state.keys(): + self.check_state_key(key) + instance.__dict__.update(state) + elif state: + slotstate.update(state) + for key, value in slotstate.items(): + if not unsafe: + self.check_state_key(key) + setattr(instance, key, value) + + def construct_python_object(self, suffix, node): + # Format: + # !!python/object:module.name { ... state ... } + instance = self.make_python_instance(suffix, node, newobj=True) + yield instance + deep = hasattr(instance, '__setstate__') + state = self.construct_mapping(node, deep=deep) + self.set_python_instance_state(instance, state) + + def construct_python_object_apply(self, suffix, node, newobj=False): + # Format: + # !!python/object/apply # (or !!python/object/new) + # args: [ ... arguments ... ] + # kwds: { ... keywords ... } + # state: ... state ... + # listitems: [ ... listitems ... ] + # dictitems: { ... dictitems ... } + # or short format: + # !!python/object/apply [ ... arguments ... ] + # The difference between !!python/object/apply and !!python/object/new + # is how an object is created, check make_python_instance for details. + if isinstance(node, SequenceNode): + args = self.construct_sequence(node, deep=True) + kwds = {} + state = {} + listitems = [] + dictitems = {} + else: + value = self.construct_mapping(node, deep=True) + args = value.get('args', []) + kwds = value.get('kwds', {}) + state = value.get('state', {}) + listitems = value.get('listitems', []) + dictitems = value.get('dictitems', {}) + instance = self.make_python_instance(suffix, node, args, kwds, newobj) + if state: + self.set_python_instance_state(instance, state) + if listitems: + instance.extend(listitems) + if dictitems: + for key in dictitems: + instance[key] = dictitems[key] + return instance + + def construct_python_object_new(self, suffix, node): + return self.construct_python_object_apply(suffix, node, newobj=True) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/none', + FullConstructor.construct_yaml_null) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/bool', + FullConstructor.construct_yaml_bool) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/str', + FullConstructor.construct_python_str) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/unicode', + FullConstructor.construct_python_unicode) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/bytes', + FullConstructor.construct_python_bytes) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/int', + FullConstructor.construct_yaml_int) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/long', + FullConstructor.construct_python_long) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/float', + FullConstructor.construct_yaml_float) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/complex', + FullConstructor.construct_python_complex) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/list', + FullConstructor.construct_yaml_seq) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/tuple', + FullConstructor.construct_python_tuple) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/dict', + FullConstructor.construct_yaml_map) + +FullConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/name:', + FullConstructor.construct_python_name) + +FullConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/module:', + FullConstructor.construct_python_module) + +FullConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object:', + FullConstructor.construct_python_object) + +FullConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object/new:', + FullConstructor.construct_python_object_new) + +class UnsafeConstructor(FullConstructor): + + def find_python_module(self, name, mark): + return super(UnsafeConstructor, self).find_python_module(name, mark, unsafe=True) + + def find_python_name(self, name, mark): + return super(UnsafeConstructor, self).find_python_name(name, mark, unsafe=True) + + def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False): + return super(UnsafeConstructor, self).make_python_instance( + suffix, node, args, kwds, newobj, unsafe=True) + + def set_python_instance_state(self, instance, state): + return super(UnsafeConstructor, self).set_python_instance_state( + instance, state, unsafe=True) + +UnsafeConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object/apply:', + UnsafeConstructor.construct_python_object_apply) + +# Constructor is same as UnsafeConstructor. Need to leave this in place in case +# people have extended it directly. +class Constructor(UnsafeConstructor): + pass diff --git a/pipenv/patched/yaml3/cyaml.py b/pipenv/patched/yaml3/cyaml.py new file mode 100644 index 00000000..1e606c74 --- /dev/null +++ b/pipenv/patched/yaml3/cyaml.py @@ -0,0 +1,101 @@ + +__all__ = [ + 'CBaseLoader', 'CSafeLoader', 'CFullLoader', 'CUnsafeLoader', 'CLoader', + 'CBaseDumper', 'CSafeDumper', 'CDumper' +] + +from _yaml import CParser, CEmitter + +from .constructor import * + +from .serializer import * +from .representer import * + +from .resolver import * + +class CBaseLoader(CParser, BaseConstructor, BaseResolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + BaseConstructor.__init__(self) + BaseResolver.__init__(self) + +class CSafeLoader(CParser, SafeConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + SafeConstructor.__init__(self) + Resolver.__init__(self) + +class CFullLoader(CParser, FullConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + FullConstructor.__init__(self) + Resolver.__init__(self) + +class CUnsafeLoader(CParser, UnsafeConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + UnsafeConstructor.__init__(self) + Resolver.__init__(self) + +class CLoader(CParser, Constructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + Constructor.__init__(self) + Resolver.__init__(self) + +class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class CSafeDumper(CEmitter, SafeRepresenter, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + SafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class CDumper(CEmitter, Serializer, Representer, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + diff --git a/pipenv/patched/yaml3/dumper.py b/pipenv/patched/yaml3/dumper.py new file mode 100644 index 00000000..6aadba55 --- /dev/null +++ b/pipenv/patched/yaml3/dumper.py @@ -0,0 +1,62 @@ + +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper'] + +from .emitter import * +from .serializer import * +from .representer import * +from .resolver import * + +class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + SafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class Dumper(Emitter, Serializer, Representer, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + diff --git a/pipenv/patched/yaml3/emitter.py b/pipenv/patched/yaml3/emitter.py new file mode 100644 index 00000000..a664d011 --- /dev/null +++ b/pipenv/patched/yaml3/emitter.py @@ -0,0 +1,1137 @@ + +# Emitter expects events obeying the following grammar: +# stream ::= STREAM-START document* STREAM-END +# document ::= DOCUMENT-START node DOCUMENT-END +# node ::= SCALAR | sequence | mapping +# sequence ::= SEQUENCE-START node* SEQUENCE-END +# mapping ::= MAPPING-START (node node)* MAPPING-END + +__all__ = ['Emitter', 'EmitterError'] + +from .error import YAMLError +from .events import * + +class EmitterError(YAMLError): + pass + +class ScalarAnalysis: + def __init__(self, scalar, empty, multiline, + allow_flow_plain, allow_block_plain, + allow_single_quoted, allow_double_quoted, + allow_block): + self.scalar = scalar + self.empty = empty + self.multiline = multiline + self.allow_flow_plain = allow_flow_plain + self.allow_block_plain = allow_block_plain + self.allow_single_quoted = allow_single_quoted + self.allow_double_quoted = allow_double_quoted + self.allow_block = allow_block + +class Emitter: + + DEFAULT_TAG_PREFIXES = { + '!' : '!', + 'tag:yaml.org,2002:' : '!!', + } + + def __init__(self, stream, canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + + # The stream should have the methods `write` and possibly `flush`. + self.stream = stream + + # Encoding can be overridden by STREAM-START. + self.encoding = None + + # Emitter is a state machine with a stack of states to handle nested + # structures. + self.states = [] + self.state = self.expect_stream_start + + # Current event and the event queue. + self.events = [] + self.event = None + + # The current indentation level and the stack of previous indents. + self.indents = [] + self.indent = None + + # Flow level. + self.flow_level = 0 + + # Contexts. + self.root_context = False + self.sequence_context = False + self.mapping_context = False + self.simple_key_context = False + + # Characteristics of the last emitted character: + # - current position. + # - is it a whitespace? + # - is it an indention character + # (indentation space, '-', '?', or ':')? + self.line = 0 + self.column = 0 + self.whitespace = True + self.indention = True + + # Whether the document requires an explicit document indicator + self.open_ended = False + + # Formatting details. + self.canonical = canonical + self.allow_unicode = allow_unicode + self.best_indent = 2 + if indent and 1 < indent < 10: + self.best_indent = indent + self.best_width = 80 + if width and width > self.best_indent*2: + self.best_width = width + self.best_line_break = '\n' + if line_break in ['\r', '\n', '\r\n']: + self.best_line_break = line_break + + # Tag prefixes. + self.tag_prefixes = None + + # Prepared anchor and tag. + self.prepared_anchor = None + self.prepared_tag = None + + # Scalar analysis and style. + self.analysis = None + self.style = None + + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + + def emit(self, event): + self.events.append(event) + while not self.need_more_events(): + self.event = self.events.pop(0) + self.state() + self.event = None + + # In some cases, we wait for a few next events before emitting. + + def need_more_events(self): + if not self.events: + return True + event = self.events[0] + if isinstance(event, DocumentStartEvent): + return self.need_events(1) + elif isinstance(event, SequenceStartEvent): + return self.need_events(2) + elif isinstance(event, MappingStartEvent): + return self.need_events(3) + else: + return False + + def need_events(self, count): + level = 0 + for event in self.events[1:]: + if isinstance(event, (DocumentStartEvent, CollectionStartEvent)): + level += 1 + elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)): + level -= 1 + elif isinstance(event, StreamEndEvent): + level = -1 + if level < 0: + return False + return (len(self.events) < count+1) + + def increase_indent(self, flow=False, indentless=False): + self.indents.append(self.indent) + if self.indent is None: + if flow: + self.indent = self.best_indent + else: + self.indent = 0 + elif not indentless: + self.indent += self.best_indent + + # States. + + # Stream handlers. + + def expect_stream_start(self): + if isinstance(self.event, StreamStartEvent): + if self.event.encoding and not hasattr(self.stream, 'encoding'): + self.encoding = self.event.encoding + self.write_stream_start() + self.state = self.expect_first_document_start + else: + raise EmitterError("expected StreamStartEvent, but got %s" + % self.event) + + def expect_nothing(self): + raise EmitterError("expected nothing, but got %s" % self.event) + + # Document handlers. + + def expect_first_document_start(self): + return self.expect_document_start(first=True) + + def expect_document_start(self, first=False): + if isinstance(self.event, DocumentStartEvent): + if (self.event.version or self.event.tags) and self.open_ended: + self.write_indicator('...', True) + self.write_indent() + if self.event.version: + version_text = self.prepare_version(self.event.version) + self.write_version_directive(version_text) + self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy() + if self.event.tags: + handles = sorted(self.event.tags.keys()) + for handle in handles: + prefix = self.event.tags[handle] + self.tag_prefixes[prefix] = handle + handle_text = self.prepare_tag_handle(handle) + prefix_text = self.prepare_tag_prefix(prefix) + self.write_tag_directive(handle_text, prefix_text) + implicit = (first and not self.event.explicit and not self.canonical + and not self.event.version and not self.event.tags + and not self.check_empty_document()) + if not implicit: + self.write_indent() + self.write_indicator('---', True) + if self.canonical: + self.write_indent() + self.state = self.expect_document_root + elif isinstance(self.event, StreamEndEvent): + if self.open_ended: + self.write_indicator('...', True) + self.write_indent() + self.write_stream_end() + self.state = self.expect_nothing + else: + raise EmitterError("expected DocumentStartEvent, but got %s" + % self.event) + + def expect_document_end(self): + if isinstance(self.event, DocumentEndEvent): + self.write_indent() + if self.event.explicit: + self.write_indicator('...', True) + self.write_indent() + self.flush_stream() + self.state = self.expect_document_start + else: + raise EmitterError("expected DocumentEndEvent, but got %s" + % self.event) + + def expect_document_root(self): + self.states.append(self.expect_document_end) + self.expect_node(root=True) + + # Node handlers. + + def expect_node(self, root=False, sequence=False, mapping=False, + simple_key=False): + self.root_context = root + self.sequence_context = sequence + self.mapping_context = mapping + self.simple_key_context = simple_key + if isinstance(self.event, AliasEvent): + self.expect_alias() + elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)): + self.process_anchor('&') + self.process_tag() + if isinstance(self.event, ScalarEvent): + self.expect_scalar() + elif isinstance(self.event, SequenceStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_sequence(): + self.expect_flow_sequence() + else: + self.expect_block_sequence() + elif isinstance(self.event, MappingStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_mapping(): + self.expect_flow_mapping() + else: + self.expect_block_mapping() + else: + raise EmitterError("expected NodeEvent, but got %s" % self.event) + + def expect_alias(self): + if self.event.anchor is None: + raise EmitterError("anchor is not specified for alias") + self.process_anchor('*') + self.state = self.states.pop() + + def expect_scalar(self): + self.increase_indent(flow=True) + self.process_scalar() + self.indent = self.indents.pop() + self.state = self.states.pop() + + # Flow sequence handlers. + + def expect_flow_sequence(self): + self.write_indicator('[', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_sequence_item + + def expect_first_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator(']', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + def expect_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(',', False) + self.write_indent() + self.write_indicator(']', False) + self.state = self.states.pop() + else: + self.write_indicator(',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + # Flow mapping handlers. + + def expect_flow_mapping(self): + self.write_indicator('{', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_mapping_key + + def expect_first_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator('}', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator('?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(',', False) + self.write_indent() + self.write_indicator('}', False) + self.state = self.states.pop() + else: + self.write_indicator(',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator('?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_simple_value(self): + self.write_indicator(':', False) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + def expect_flow_mapping_value(self): + if self.canonical or self.column > self.best_width: + self.write_indent() + self.write_indicator(':', True) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + # Block sequence handlers. + + def expect_block_sequence(self): + indentless = (self.mapping_context and not self.indention) + self.increase_indent(flow=False, indentless=indentless) + self.state = self.expect_first_block_sequence_item + + def expect_first_block_sequence_item(self): + return self.expect_block_sequence_item(first=True) + + def expect_block_sequence_item(self, first=False): + if not first and isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + self.write_indicator('-', True, indention=True) + self.states.append(self.expect_block_sequence_item) + self.expect_node(sequence=True) + + # Block mapping handlers. + + def expect_block_mapping(self): + self.increase_indent(flow=False) + self.state = self.expect_first_block_mapping_key + + def expect_first_block_mapping_key(self): + return self.expect_block_mapping_key(first=True) + + def expect_block_mapping_key(self, first=False): + if not first and isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + if self.check_simple_key(): + self.states.append(self.expect_block_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator('?', True, indention=True) + self.states.append(self.expect_block_mapping_value) + self.expect_node(mapping=True) + + def expect_block_mapping_simple_value(self): + self.write_indicator(':', False) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + def expect_block_mapping_value(self): + self.write_indent() + self.write_indicator(':', True, indention=True) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + # Checkers. + + def check_empty_sequence(self): + return (isinstance(self.event, SequenceStartEvent) and self.events + and isinstance(self.events[0], SequenceEndEvent)) + + def check_empty_mapping(self): + return (isinstance(self.event, MappingStartEvent) and self.events + and isinstance(self.events[0], MappingEndEvent)) + + def check_empty_document(self): + if not isinstance(self.event, DocumentStartEvent) or not self.events: + return False + event = self.events[0] + return (isinstance(event, ScalarEvent) and event.anchor is None + and event.tag is None and event.implicit and event.value == '') + + def check_simple_key(self): + length = 0 + if isinstance(self.event, NodeEvent) and self.event.anchor is not None: + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + length += len(self.prepared_anchor) + if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \ + and self.event.tag is not None: + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(self.event.tag) + length += len(self.prepared_tag) + if isinstance(self.event, ScalarEvent): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + length += len(self.analysis.scalar) + return (length < 128 and (isinstance(self.event, AliasEvent) + or (isinstance(self.event, ScalarEvent) + and not self.analysis.empty and not self.analysis.multiline) + or self.check_empty_sequence() or self.check_empty_mapping())) + + # Anchor, Tag, and Scalar processors. + + def process_anchor(self, indicator): + if self.event.anchor is None: + self.prepared_anchor = None + return + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + if self.prepared_anchor: + self.write_indicator(indicator+self.prepared_anchor, True) + self.prepared_anchor = None + + def process_tag(self): + tag = self.event.tag + if isinstance(self.event, ScalarEvent): + if self.style is None: + self.style = self.choose_scalar_style() + if ((not self.canonical or tag is None) and + ((self.style == '' and self.event.implicit[0]) + or (self.style != '' and self.event.implicit[1]))): + self.prepared_tag = None + return + if self.event.implicit[0] and tag is None: + tag = '!' + self.prepared_tag = None + else: + if (not self.canonical or tag is None) and self.event.implicit: + self.prepared_tag = None + return + if tag is None: + raise EmitterError("tag is not specified") + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(tag) + if self.prepared_tag: + self.write_indicator(self.prepared_tag, True) + self.prepared_tag = None + + def choose_scalar_style(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.event.style == '"' or self.canonical: + return '"' + if not self.event.style and self.event.implicit[0]: + if (not (self.simple_key_context and + (self.analysis.empty or self.analysis.multiline)) + and (self.flow_level and self.analysis.allow_flow_plain + or (not self.flow_level and self.analysis.allow_block_plain))): + return '' + if self.event.style and self.event.style in '|>': + if (not self.flow_level and not self.simple_key_context + and self.analysis.allow_block): + return self.event.style + if not self.event.style or self.event.style == '\'': + if (self.analysis.allow_single_quoted and + not (self.simple_key_context and self.analysis.multiline)): + return '\'' + return '"' + + def process_scalar(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.style is None: + self.style = self.choose_scalar_style() + split = (not self.simple_key_context) + #if self.analysis.multiline and split \ + # and (not self.style or self.style in '\'\"'): + # self.write_indent() + if self.style == '"': + self.write_double_quoted(self.analysis.scalar, split) + elif self.style == '\'': + self.write_single_quoted(self.analysis.scalar, split) + elif self.style == '>': + self.write_folded(self.analysis.scalar) + elif self.style == '|': + self.write_literal(self.analysis.scalar) + else: + self.write_plain(self.analysis.scalar, split) + self.analysis = None + self.style = None + + # Analyzers. + + def prepare_version(self, version): + major, minor = version + if major != 1: + raise EmitterError("unsupported YAML version: %d.%d" % (major, minor)) + return '%d.%d' % (major, minor) + + def prepare_tag_handle(self, handle): + if not handle: + raise EmitterError("tag handle must not be empty") + if handle[0] != '!' or handle[-1] != '!': + raise EmitterError("tag handle must start and end with '!': %r" % handle) + for ch in handle[1:-1]: + if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_'): + raise EmitterError("invalid character %r in the tag handle: %r" + % (ch, handle)) + return handle + + def prepare_tag_prefix(self, prefix): + if not prefix: + raise EmitterError("tag prefix must not be empty") + chunks = [] + start = end = 0 + if prefix[0] == '!': + end = 1 + while end < len(prefix): + ch = prefix[end] + if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-;/?!:@&=+$,_.~*\'()[]': + end += 1 + else: + if start < end: + chunks.append(prefix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append('%%%02X' % ord(ch)) + if start < end: + chunks.append(prefix[start:end]) + return ''.join(chunks) + + def prepare_tag(self, tag): + if not tag: + raise EmitterError("tag must not be empty") + if tag == '!': + return tag + handle = None + suffix = tag + prefixes = sorted(self.tag_prefixes.keys()) + for prefix in prefixes: + if tag.startswith(prefix) \ + and (prefix == '!' or len(prefix) < len(tag)): + handle = self.tag_prefixes[prefix] + suffix = tag[len(prefix):] + chunks = [] + start = end = 0 + while end < len(suffix): + ch = suffix[end] + if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-;/?:@&=+$,_.~*\'()[]' \ + or (ch == '!' and handle != '!'): + end += 1 + else: + if start < end: + chunks.append(suffix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append('%%%02X' % ch) + if start < end: + chunks.append(suffix[start:end]) + suffix_text = ''.join(chunks) + if handle: + return '%s%s' % (handle, suffix_text) + else: + return '!<%s>' % suffix_text + + def prepare_anchor(self, anchor): + if not anchor: + raise EmitterError("anchor must not be empty") + for ch in anchor: + if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_'): + raise EmitterError("invalid character %r in the anchor: %r" + % (ch, anchor)) + return anchor + + def analyze_scalar(self, scalar): + + # Empty scalar is a special case. + if not scalar: + return ScalarAnalysis(scalar=scalar, empty=True, multiline=False, + allow_flow_plain=False, allow_block_plain=True, + allow_single_quoted=True, allow_double_quoted=True, + allow_block=False) + + # Indicators and special characters. + block_indicators = False + flow_indicators = False + line_breaks = False + special_characters = False + + # Important whitespace combinations. + leading_space = False + leading_break = False + trailing_space = False + trailing_break = False + break_space = False + space_break = False + + # Check document indicators. + if scalar.startswith('---') or scalar.startswith('...'): + block_indicators = True + flow_indicators = True + + # First character or preceded by a whitespace. + preceded_by_whitespace = True + + # Last character or followed by a whitespace. + followed_by_whitespace = (len(scalar) == 1 or + scalar[1] in '\0 \t\r\n\x85\u2028\u2029') + + # The previous character is a space. + previous_space = False + + # The previous character is a break. + previous_break = False + + index = 0 + while index < len(scalar): + ch = scalar[index] + + # Check for indicators. + if index == 0: + # Leading indicators are special characters. + if ch in '#,[]{}&*!|>\'\"%@`': + flow_indicators = True + block_indicators = True + if ch in '?:': + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == '-' and followed_by_whitespace: + flow_indicators = True + block_indicators = True + else: + # Some indicators cannot appear within a scalar as well. + if ch in ',?[]{}': + flow_indicators = True + if ch == ':': + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == '#' and preceded_by_whitespace: + flow_indicators = True + block_indicators = True + + # Check for line breaks, special, and unicode characters. + if ch in '\n\x85\u2028\u2029': + line_breaks = True + if not (ch == '\n' or '\x20' <= ch <= '\x7E'): + if (ch == '\x85' or '\xA0' <= ch <= '\uD7FF' + or '\uE000' <= ch <= '\uFFFD' + or '\U00010000' <= ch < '\U0010ffff') and ch != '\uFEFF': + unicode_characters = True + if not self.allow_unicode: + special_characters = True + else: + special_characters = True + + # Detect important whitespace combinations. + if ch == ' ': + if index == 0: + leading_space = True + if index == len(scalar)-1: + trailing_space = True + if previous_break: + break_space = True + previous_space = True + previous_break = False + elif ch in '\n\x85\u2028\u2029': + if index == 0: + leading_break = True + if index == len(scalar)-1: + trailing_break = True + if previous_space: + space_break = True + previous_space = False + previous_break = True + else: + previous_space = False + previous_break = False + + # Prepare for the next character. + index += 1 + preceded_by_whitespace = (ch in '\0 \t\r\n\x85\u2028\u2029') + followed_by_whitespace = (index+1 >= len(scalar) or + scalar[index+1] in '\0 \t\r\n\x85\u2028\u2029') + + # Let's decide what styles are allowed. + allow_flow_plain = True + allow_block_plain = True + allow_single_quoted = True + allow_double_quoted = True + allow_block = True + + # Leading and trailing whitespaces are bad for plain scalars. + if (leading_space or leading_break + or trailing_space or trailing_break): + allow_flow_plain = allow_block_plain = False + + # We do not permit trailing spaces for block scalars. + if trailing_space: + allow_block = False + + # Spaces at the beginning of a new line are only acceptable for block + # scalars. + if break_space: + allow_flow_plain = allow_block_plain = allow_single_quoted = False + + # Spaces followed by breaks, as well as special character are only + # allowed for double quoted scalars. + if space_break or special_characters: + allow_flow_plain = allow_block_plain = \ + allow_single_quoted = allow_block = False + + # Although the plain scalar writer supports breaks, we never emit + # multiline plain scalars. + if line_breaks: + allow_flow_plain = allow_block_plain = False + + # Flow indicators are forbidden for flow plain scalars. + if flow_indicators: + allow_flow_plain = False + + # Block indicators are forbidden for block plain scalars. + if block_indicators: + allow_block_plain = False + + return ScalarAnalysis(scalar=scalar, + empty=False, multiline=line_breaks, + allow_flow_plain=allow_flow_plain, + allow_block_plain=allow_block_plain, + allow_single_quoted=allow_single_quoted, + allow_double_quoted=allow_double_quoted, + allow_block=allow_block) + + # Writers. + + def flush_stream(self): + if hasattr(self.stream, 'flush'): + self.stream.flush() + + def write_stream_start(self): + # Write BOM if needed. + if self.encoding and self.encoding.startswith('utf-16'): + self.stream.write('\uFEFF'.encode(self.encoding)) + + def write_stream_end(self): + self.flush_stream() + + def write_indicator(self, indicator, need_whitespace, + whitespace=False, indention=False): + if self.whitespace or not need_whitespace: + data = indicator + else: + data = ' '+indicator + self.whitespace = whitespace + self.indention = self.indention and indention + self.column += len(data) + self.open_ended = False + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_indent(self): + indent = self.indent or 0 + if not self.indention or self.column > indent \ + or (self.column == indent and not self.whitespace): + self.write_line_break() + if self.column < indent: + self.whitespace = True + data = ' '*(indent-self.column) + self.column = indent + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_line_break(self, data=None): + if data is None: + data = self.best_line_break + self.whitespace = True + self.indention = True + self.line += 1 + self.column = 0 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_version_directive(self, version_text): + data = '%%YAML %s' % version_text + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + def write_tag_directive(self, handle_text, prefix_text): + data = '%%TAG %s %s' % (handle_text, prefix_text) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + # Scalar streams. + + def write_single_quoted(self, text, split=True): + self.write_indicator('\'', True) + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch is None or ch != ' ': + if start+1 == end and self.column > self.best_width and split \ + and start != 0 and end != len(text): + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch is None or ch not in '\n\x85\u2028\u2029': + if text[start] == '\n': + self.write_line_break() + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + start = end + else: + if ch is None or ch in ' \n\x85\u2028\u2029' or ch == '\'': + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch == '\'': + data = '\'\'' + self.column += 2 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + 1 + if ch is not None: + spaces = (ch == ' ') + breaks = (ch in '\n\x85\u2028\u2029') + end += 1 + self.write_indicator('\'', False) + + ESCAPE_REPLACEMENTS = { + '\0': '0', + '\x07': 'a', + '\x08': 'b', + '\x09': 't', + '\x0A': 'n', + '\x0B': 'v', + '\x0C': 'f', + '\x0D': 'r', + '\x1B': 'e', + '\"': '\"', + '\\': '\\', + '\x85': 'N', + '\xA0': '_', + '\u2028': 'L', + '\u2029': 'P', + } + + def write_double_quoted(self, text, split=True): + self.write_indicator('"', True) + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if ch is None or ch in '"\\\x85\u2028\u2029\uFEFF' \ + or not ('\x20' <= ch <= '\x7E' + or (self.allow_unicode + and ('\xA0' <= ch <= '\uD7FF' + or '\uE000' <= ch <= '\uFFFD'))): + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + if ch in self.ESCAPE_REPLACEMENTS: + data = '\\'+self.ESCAPE_REPLACEMENTS[ch] + elif ch <= '\xFF': + data = '\\x%02X' % ord(ch) + elif ch <= '\uFFFF': + data = '\\u%04X' % ord(ch) + else: + data = '\\U%08X' % ord(ch) + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end+1 + if 0 < end < len(text)-1 and (ch == ' ' or start >= end) \ + and self.column+(end-start) > self.best_width and split: + data = text[start:end]+'\\' + if start < end: + start = end + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_indent() + self.whitespace = False + self.indention = False + if text[start] == ' ': + data = '\\' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + end += 1 + self.write_indicator('"', False) + + def determine_block_hints(self, text): + hints = '' + if text: + if text[0] in ' \n\x85\u2028\u2029': + hints += str(self.best_indent) + if text[-1] not in '\n\x85\u2028\u2029': + hints += '-' + elif len(text) == 1 or text[-2] in '\n\x85\u2028\u2029': + hints += '+' + return hints + + def write_folded(self, text): + hints = self.determine_block_hints(text) + self.write_indicator('>'+hints, True) + if hints[-1:] == '+': + self.open_ended = True + self.write_line_break() + leading_space = True + spaces = False + breaks = True + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in '\n\x85\u2028\u2029': + if not leading_space and ch is not None and ch != ' ' \ + and text[start] == '\n': + self.write_line_break() + leading_space = (ch == ' ') + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + elif spaces: + if ch != ' ': + if start+1 == end and self.column > self.best_width: + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + else: + if ch is None or ch in ' \n\x85\u2028\u2029': + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in '\n\x85\u2028\u2029') + spaces = (ch == ' ') + end += 1 + + def write_literal(self, text): + hints = self.determine_block_hints(text) + self.write_indicator('|'+hints, True) + if hints[-1:] == '+': + self.open_ended = True + self.write_line_break() + breaks = True + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in '\n\x85\u2028\u2029': + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + else: + if ch is None or ch in '\n\x85\u2028\u2029': + data = text[start:end] + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in '\n\x85\u2028\u2029') + end += 1 + + def write_plain(self, text, split=True): + if self.root_context: + self.open_ended = True + if not text: + return + if not self.whitespace: + data = ' ' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.whitespace = False + self.indention = False + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch != ' ': + if start+1 == end and self.column > self.best_width and split: + self.write_indent() + self.whitespace = False + self.indention = False + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch not in '\n\x85\u2028\u2029': + if text[start] == '\n': + self.write_line_break() + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + self.whitespace = False + self.indention = False + start = end + else: + if ch is None or ch in ' \n\x85\u2028\u2029': + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + spaces = (ch == ' ') + breaks = (ch in '\n\x85\u2028\u2029') + end += 1 diff --git a/pipenv/patched/yaml3/error.py b/pipenv/patched/yaml3/error.py new file mode 100644 index 00000000..b796b4dc --- /dev/null +++ b/pipenv/patched/yaml3/error.py @@ -0,0 +1,75 @@ + +__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError'] + +class Mark: + + def __init__(self, name, index, line, column, buffer, pointer): + self.name = name + self.index = index + self.line = line + self.column = column + self.buffer = buffer + self.pointer = pointer + + def get_snippet(self, indent=4, max_length=75): + if self.buffer is None: + return None + head = '' + start = self.pointer + while start > 0 and self.buffer[start-1] not in '\0\r\n\x85\u2028\u2029': + start -= 1 + if self.pointer-start > max_length/2-1: + head = ' ... ' + start += 5 + break + tail = '' + end = self.pointer + while end < len(self.buffer) and self.buffer[end] not in '\0\r\n\x85\u2028\u2029': + end += 1 + if end-self.pointer > max_length/2-1: + tail = ' ... ' + end -= 5 + break + snippet = self.buffer[start:end] + return ' '*indent + head + snippet + tail + '\n' \ + + ' '*(indent+self.pointer-start+len(head)) + '^' + + def __str__(self): + snippet = self.get_snippet() + where = " in \"%s\", line %d, column %d" \ + % (self.name, self.line+1, self.column+1) + if snippet is not None: + where += ":\n"+snippet + return where + +class YAMLError(Exception): + pass + +class MarkedYAMLError(YAMLError): + + def __init__(self, context=None, context_mark=None, + problem=None, problem_mark=None, note=None): + self.context = context + self.context_mark = context_mark + self.problem = problem + self.problem_mark = problem_mark + self.note = note + + def __str__(self): + lines = [] + if self.context is not None: + lines.append(self.context) + if self.context_mark is not None \ + and (self.problem is None or self.problem_mark is None + or self.context_mark.name != self.problem_mark.name + or self.context_mark.line != self.problem_mark.line + or self.context_mark.column != self.problem_mark.column): + lines.append(str(self.context_mark)) + if self.problem is not None: + lines.append(self.problem) + if self.problem_mark is not None: + lines.append(str(self.problem_mark)) + if self.note is not None: + lines.append(self.note) + return '\n'.join(lines) + diff --git a/pipenv/patched/yaml3/events.py b/pipenv/patched/yaml3/events.py new file mode 100644 index 00000000..f79ad389 --- /dev/null +++ b/pipenv/patched/yaml3/events.py @@ -0,0 +1,86 @@ + +# Abstract classes. + +class Event(object): + def __init__(self, start_mark=None, end_mark=None): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in ['anchor', 'tag', 'implicit', 'value'] + if hasattr(self, key)] + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +class NodeEvent(Event): + def __init__(self, anchor, start_mark=None, end_mark=None): + self.anchor = anchor + self.start_mark = start_mark + self.end_mark = end_mark + +class CollectionStartEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None, + flow_style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class CollectionEndEvent(Event): + pass + +# Implementations. + +class StreamStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndEvent(Event): + pass + +class DocumentStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None, version=None, tags=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + self.version = version + self.tags = tags + +class DocumentEndEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + +class AliasEvent(NodeEvent): + pass + +class ScalarEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, value, + start_mark=None, end_mark=None, style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class SequenceStartEvent(CollectionStartEvent): + pass + +class SequenceEndEvent(CollectionEndEvent): + pass + +class MappingStartEvent(CollectionStartEvent): + pass + +class MappingEndEvent(CollectionEndEvent): + pass + diff --git a/pipenv/patched/yaml3/loader.py b/pipenv/patched/yaml3/loader.py new file mode 100644 index 00000000..e90c1122 --- /dev/null +++ b/pipenv/patched/yaml3/loader.py @@ -0,0 +1,63 @@ + +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader'] + +from .reader import * +from .scanner import * +from .parser import * +from .composer import * +from .constructor import * +from .resolver import * + +class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + BaseConstructor.__init__(self) + BaseResolver.__init__(self) + +class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + FullConstructor.__init__(self) + Resolver.__init__(self) + +class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + SafeConstructor.__init__(self) + Resolver.__init__(self) + +class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) + +# UnsafeLoader is the same as Loader (which is and was always unsafe on +# untrusted input). Use of either Loader or UnsafeLoader should be rare, since +# FullLoad should be able to load almost all YAML safely. Loader is left intact +# to ensure backwards compatibility. +class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) diff --git a/pipenv/patched/yaml3/nodes.py b/pipenv/patched/yaml3/nodes.py new file mode 100644 index 00000000..c4f070c4 --- /dev/null +++ b/pipenv/patched/yaml3/nodes.py @@ -0,0 +1,49 @@ + +class Node(object): + def __init__(self, tag, value, start_mark, end_mark): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + value = self.value + #if isinstance(value, list): + # if len(value) == 0: + # value = '' + # elif len(value) == 1: + # value = '<1 item>' + # else: + # value = '<%d items>' % len(value) + #else: + # if len(value) > 75: + # value = repr(value[:70]+u' ... ') + # else: + # value = repr(value) + value = repr(value) + return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value) + +class ScalarNode(Node): + id = 'scalar' + def __init__(self, tag, value, + start_mark=None, end_mark=None, style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class CollectionNode(Node): + def __init__(self, tag, value, + start_mark=None, end_mark=None, flow_style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class SequenceNode(CollectionNode): + id = 'sequence' + +class MappingNode(CollectionNode): + id = 'mapping' + diff --git a/pipenv/patched/yaml3/parser.py b/pipenv/patched/yaml3/parser.py new file mode 100644 index 00000000..13a5995d --- /dev/null +++ b/pipenv/patched/yaml3/parser.py @@ -0,0 +1,589 @@ + +# The following YAML grammar is LL(1) and is parsed by a recursive descent +# parser. +# +# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +# implicit_document ::= block_node DOCUMENT-END* +# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +# block_node_or_indentless_sequence ::= +# ALIAS +# | properties (block_content | indentless_block_sequence)? +# | block_content +# | indentless_block_sequence +# block_node ::= ALIAS +# | properties block_content? +# | block_content +# flow_node ::= ALIAS +# | properties flow_content? +# | flow_content +# properties ::= TAG ANCHOR? | ANCHOR TAG? +# block_content ::= block_collection | flow_collection | SCALAR +# flow_content ::= flow_collection | SCALAR +# block_collection ::= block_sequence | block_mapping +# flow_collection ::= flow_sequence | flow_mapping +# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +# indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +# block_mapping ::= BLOCK-MAPPING_START +# ((KEY block_node_or_indentless_sequence?)? +# (VALUE block_node_or_indentless_sequence?)?)* +# BLOCK-END +# flow_sequence ::= FLOW-SEQUENCE-START +# (flow_sequence_entry FLOW-ENTRY)* +# flow_sequence_entry? +# FLOW-SEQUENCE-END +# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# flow_mapping ::= FLOW-MAPPING-START +# (flow_mapping_entry FLOW-ENTRY)* +# flow_mapping_entry? +# FLOW-MAPPING-END +# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# +# FIRST sets: +# +# stream: { STREAM-START } +# explicit_document: { DIRECTIVE DOCUMENT-START } +# implicit_document: FIRST(block_node) +# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_sequence: { BLOCK-SEQUENCE-START } +# block_mapping: { BLOCK-MAPPING-START } +# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY } +# indentless_sequence: { ENTRY } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_sequence: { FLOW-SEQUENCE-START } +# flow_mapping: { FLOW-MAPPING-START } +# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } +# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } + +__all__ = ['Parser', 'ParserError'] + +from .error import MarkedYAMLError +from .tokens import * +from .events import * +from .scanner import * + +class ParserError(MarkedYAMLError): + pass + +class Parser: + # Since writing a recursive-descendant parser is a straightforward task, we + # do not give many comments here. + + DEFAULT_TAGS = { + '!': '!', + '!!': 'tag:yaml.org,2002:', + } + + def __init__(self): + self.current_event = None + self.yaml_version = None + self.tag_handles = {} + self.states = [] + self.marks = [] + self.state = self.parse_stream_start + + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + + def check_event(self, *choices): + # Check the type of the next event. + if self.current_event is None: + if self.state: + self.current_event = self.state() + if self.current_event is not None: + if not choices: + return True + for choice in choices: + if isinstance(self.current_event, choice): + return True + return False + + def peek_event(self): + # Get the next event. + if self.current_event is None: + if self.state: + self.current_event = self.state() + return self.current_event + + def get_event(self): + # Get the next event and proceed further. + if self.current_event is None: + if self.state: + self.current_event = self.state() + value = self.current_event + self.current_event = None + return value + + # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END + # implicit_document ::= block_node DOCUMENT-END* + # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* + + def parse_stream_start(self): + + # Parse the stream start. + token = self.get_token() + event = StreamStartEvent(token.start_mark, token.end_mark, + encoding=token.encoding) + + # Prepare the next state. + self.state = self.parse_implicit_document_start + + return event + + def parse_implicit_document_start(self): + + # Parse an implicit document. + if not self.check_token(DirectiveToken, DocumentStartToken, + StreamEndToken): + self.tag_handles = self.DEFAULT_TAGS + token = self.peek_token() + start_mark = end_mark = token.start_mark + event = DocumentStartEvent(start_mark, end_mark, + explicit=False) + + # Prepare the next state. + self.states.append(self.parse_document_end) + self.state = self.parse_block_node + + return event + + else: + return self.parse_document_start() + + def parse_document_start(self): + + # Parse any extra document end indicators. + while self.check_token(DocumentEndToken): + self.get_token() + + # Parse an explicit document. + if not self.check_token(StreamEndToken): + token = self.peek_token() + start_mark = token.start_mark + version, tags = self.process_directives() + if not self.check_token(DocumentStartToken): + raise ParserError(None, None, + "expected '', but found %r" + % self.peek_token().id, + self.peek_token().start_mark) + token = self.get_token() + end_mark = token.end_mark + event = DocumentStartEvent(start_mark, end_mark, + explicit=True, version=version, tags=tags) + self.states.append(self.parse_document_end) + self.state = self.parse_document_content + else: + # Parse the end of the stream. + token = self.get_token() + event = StreamEndEvent(token.start_mark, token.end_mark) + assert not self.states + assert not self.marks + self.state = None + return event + + def parse_document_end(self): + + # Parse the document end. + token = self.peek_token() + start_mark = end_mark = token.start_mark + explicit = False + if self.check_token(DocumentEndToken): + token = self.get_token() + end_mark = token.end_mark + explicit = True + event = DocumentEndEvent(start_mark, end_mark, + explicit=explicit) + + # Prepare the next state. + self.state = self.parse_document_start + + return event + + def parse_document_content(self): + if self.check_token(DirectiveToken, + DocumentStartToken, DocumentEndToken, StreamEndToken): + event = self.process_empty_scalar(self.peek_token().start_mark) + self.state = self.states.pop() + return event + else: + return self.parse_block_node() + + def process_directives(self): + self.yaml_version = None + self.tag_handles = {} + while self.check_token(DirectiveToken): + token = self.get_token() + if token.name == 'YAML': + if self.yaml_version is not None: + raise ParserError(None, None, + "found duplicate YAML directive", token.start_mark) + major, minor = token.value + if major != 1: + raise ParserError(None, None, + "found incompatible YAML document (version 1.* is required)", + token.start_mark) + self.yaml_version = token.value + elif token.name == 'TAG': + handle, prefix = token.value + if handle in self.tag_handles: + raise ParserError(None, None, + "duplicate tag handle %r" % handle, + token.start_mark) + self.tag_handles[handle] = prefix + if self.tag_handles: + value = self.yaml_version, self.tag_handles.copy() + else: + value = self.yaml_version, None + for key in self.DEFAULT_TAGS: + if key not in self.tag_handles: + self.tag_handles[key] = self.DEFAULT_TAGS[key] + return value + + # block_node_or_indentless_sequence ::= ALIAS + # | properties (block_content | indentless_block_sequence)? + # | block_content + # | indentless_block_sequence + # block_node ::= ALIAS + # | properties block_content? + # | block_content + # flow_node ::= ALIAS + # | properties flow_content? + # | flow_content + # properties ::= TAG ANCHOR? | ANCHOR TAG? + # block_content ::= block_collection | flow_collection | SCALAR + # flow_content ::= flow_collection | SCALAR + # block_collection ::= block_sequence | block_mapping + # flow_collection ::= flow_sequence | flow_mapping + + def parse_block_node(self): + return self.parse_node(block=True) + + def parse_flow_node(self): + return self.parse_node() + + def parse_block_node_or_indentless_sequence(self): + return self.parse_node(block=True, indentless_sequence=True) + + def parse_node(self, block=False, indentless_sequence=False): + if self.check_token(AliasToken): + token = self.get_token() + event = AliasEvent(token.value, token.start_mark, token.end_mark) + self.state = self.states.pop() + else: + anchor = None + tag = None + start_mark = end_mark = tag_mark = None + if self.check_token(AnchorToken): + token = self.get_token() + start_mark = token.start_mark + end_mark = token.end_mark + anchor = token.value + if self.check_token(TagToken): + token = self.get_token() + tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + elif self.check_token(TagToken): + token = self.get_token() + start_mark = tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + if self.check_token(AnchorToken): + token = self.get_token() + end_mark = token.end_mark + anchor = token.value + if tag is not None: + handle, suffix = tag + if handle is not None: + if handle not in self.tag_handles: + raise ParserError("while parsing a node", start_mark, + "found undefined tag handle %r" % handle, + tag_mark) + tag = self.tag_handles[handle]+suffix + else: + tag = suffix + #if tag == '!': + # raise ParserError("while parsing a node", start_mark, + # "found non-specific tag '!'", tag_mark, + # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.") + if start_mark is None: + start_mark = end_mark = self.peek_token().start_mark + event = None + implicit = (tag is None or tag == '!') + if indentless_sequence and self.check_token(BlockEntryToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark) + self.state = self.parse_indentless_sequence_entry + else: + if self.check_token(ScalarToken): + token = self.get_token() + end_mark = token.end_mark + if (token.plain and tag is None) or tag == '!': + implicit = (True, False) + elif tag is None: + implicit = (False, True) + else: + implicit = (False, False) + event = ScalarEvent(anchor, tag, implicit, token.value, + start_mark, end_mark, style=token.style) + self.state = self.states.pop() + elif self.check_token(FlowSequenceStartToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + self.state = self.parse_flow_sequence_first_entry + elif self.check_token(FlowMappingStartToken): + end_mark = self.peek_token().end_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + self.state = self.parse_flow_mapping_first_key + elif block and self.check_token(BlockSequenceStartToken): + end_mark = self.peek_token().start_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + self.state = self.parse_block_sequence_first_entry + elif block and self.check_token(BlockMappingStartToken): + end_mark = self.peek_token().start_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + self.state = self.parse_block_mapping_first_key + elif anchor is not None or tag is not None: + # Empty scalars are allowed even if a tag or an anchor is + # specified. + event = ScalarEvent(anchor, tag, (implicit, False), '', + start_mark, end_mark) + self.state = self.states.pop() + else: + if block: + node = 'block' + else: + node = 'flow' + token = self.peek_token() + raise ParserError("while parsing a %s node" % node, start_mark, + "expected the node content, but found %r" % token.id, + token.start_mark) + return event + + # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END + + def parse_block_sequence_first_entry(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_block_sequence_entry() + + def parse_block_sequence_entry(self): + if self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, BlockEndToken): + self.states.append(self.parse_block_sequence_entry) + return self.parse_block_node() + else: + self.state = self.parse_block_sequence_entry + return self.process_empty_scalar(token.end_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while parsing a block collection", self.marks[-1], + "expected , but found %r" % token.id, token.start_mark) + token = self.get_token() + event = SequenceEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + # indentless_sequence ::= (BLOCK-ENTRY block_node?)+ + + def parse_indentless_sequence_entry(self): + if self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, + KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_indentless_sequence_entry) + return self.parse_block_node() + else: + self.state = self.parse_indentless_sequence_entry + return self.process_empty_scalar(token.end_mark) + token = self.peek_token() + event = SequenceEndEvent(token.start_mark, token.start_mark) + self.state = self.states.pop() + return event + + # block_mapping ::= BLOCK-MAPPING_START + # ((KEY block_node_or_indentless_sequence?)? + # (VALUE block_node_or_indentless_sequence?)?)* + # BLOCK-END + + def parse_block_mapping_first_key(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_block_mapping_key() + + def parse_block_mapping_key(self): + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_block_mapping_value) + return self.parse_block_node_or_indentless_sequence() + else: + self.state = self.parse_block_mapping_value + return self.process_empty_scalar(token.end_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while parsing a block mapping", self.marks[-1], + "expected , but found %r" % token.id, token.start_mark) + token = self.get_token() + event = MappingEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_block_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_block_mapping_key) + return self.parse_block_node_or_indentless_sequence() + else: + self.state = self.parse_block_mapping_key + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_block_mapping_key + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + # flow_sequence ::= FLOW-SEQUENCE-START + # (flow_sequence_entry FLOW-ENTRY)* + # flow_sequence_entry? + # FLOW-SEQUENCE-END + # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + # + # Note that while production rules for both flow_sequence_entry and + # flow_mapping_entry are equal, their interpretations are different. + # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?` + # generate an inline mapping (set syntax). + + def parse_flow_sequence_first_entry(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_flow_sequence_entry(first=True) + + def parse_flow_sequence_entry(self, first=False): + if not self.check_token(FlowSequenceEndToken): + if not first: + if self.check_token(FlowEntryToken): + self.get_token() + else: + token = self.peek_token() + raise ParserError("while parsing a flow sequence", self.marks[-1], + "expected ',' or ']', but got %r" % token.id, token.start_mark) + + if self.check_token(KeyToken): + token = self.peek_token() + event = MappingStartEvent(None, None, True, + token.start_mark, token.end_mark, + flow_style=True) + self.state = self.parse_flow_sequence_entry_mapping_key + return event + elif not self.check_token(FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry) + return self.parse_flow_node() + token = self.get_token() + event = SequenceEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_flow_sequence_entry_mapping_key(self): + token = self.get_token() + if not self.check_token(ValueToken, + FlowEntryToken, FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry_mapping_value) + return self.parse_flow_node() + else: + self.state = self.parse_flow_sequence_entry_mapping_value + return self.process_empty_scalar(token.end_mark) + + def parse_flow_sequence_entry_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry_mapping_end) + return self.parse_flow_node() + else: + self.state = self.parse_flow_sequence_entry_mapping_end + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_flow_sequence_entry_mapping_end + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + def parse_flow_sequence_entry_mapping_end(self): + self.state = self.parse_flow_sequence_entry + token = self.peek_token() + return MappingEndEvent(token.start_mark, token.start_mark) + + # flow_mapping ::= FLOW-MAPPING-START + # (flow_mapping_entry FLOW-ENTRY)* + # flow_mapping_entry? + # FLOW-MAPPING-END + # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + + def parse_flow_mapping_first_key(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_flow_mapping_key(first=True) + + def parse_flow_mapping_key(self, first=False): + if not self.check_token(FlowMappingEndToken): + if not first: + if self.check_token(FlowEntryToken): + self.get_token() + else: + token = self.peek_token() + raise ParserError("while parsing a flow mapping", self.marks[-1], + "expected ',' or '}', but got %r" % token.id, token.start_mark) + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(ValueToken, + FlowEntryToken, FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_value) + return self.parse_flow_node() + else: + self.state = self.parse_flow_mapping_value + return self.process_empty_scalar(token.end_mark) + elif not self.check_token(FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_empty_value) + return self.parse_flow_node() + token = self.get_token() + event = MappingEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_flow_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_key) + return self.parse_flow_node() + else: + self.state = self.parse_flow_mapping_key + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_flow_mapping_key + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + def parse_flow_mapping_empty_value(self): + self.state = self.parse_flow_mapping_key + return self.process_empty_scalar(self.peek_token().start_mark) + + def process_empty_scalar(self, mark): + return ScalarEvent(None, None, (True, False), '', mark, mark) + diff --git a/pipenv/patched/yaml3/reader.py b/pipenv/patched/yaml3/reader.py new file mode 100644 index 00000000..774b0219 --- /dev/null +++ b/pipenv/patched/yaml3/reader.py @@ -0,0 +1,185 @@ +# This module contains abstractions for the input stream. You don't have to +# looks further, there are no pretty code. +# +# We define two classes here. +# +# Mark(source, line, column) +# It's just a record and its only use is producing nice error messages. +# Parser does not use it for any other purposes. +# +# Reader(source, data) +# Reader determines the encoding of `data` and converts it to unicode. +# Reader provides the following methods and attributes: +# reader.peek(length=1) - return the next `length` characters +# reader.forward(length=1) - move the current position to `length` characters. +# reader.index - the number of the current character. +# reader.line, stream.column - the line and the column of the current character. + +__all__ = ['Reader', 'ReaderError'] + +from .error import YAMLError, Mark + +import codecs, re + +class ReaderError(YAMLError): + + def __init__(self, name, position, character, encoding, reason): + self.name = name + self.character = character + self.position = position + self.encoding = encoding + self.reason = reason + + def __str__(self): + if isinstance(self.character, bytes): + return "'%s' codec can't decode byte #x%02x: %s\n" \ + " in \"%s\", position %d" \ + % (self.encoding, ord(self.character), self.reason, + self.name, self.position) + else: + return "unacceptable character #x%04x: %s\n" \ + " in \"%s\", position %d" \ + % (self.character, self.reason, + self.name, self.position) + +class Reader(object): + # Reader: + # - determines the data encoding and converts it to a unicode string, + # - checks if characters are in allowed range, + # - adds '\0' to the end. + + # Reader accepts + # - a `bytes` object, + # - a `str` object, + # - a file-like object with its `read` method returning `str`, + # - a file-like object with its `read` method returning `unicode`. + + # Yeah, it's ugly and slow. + + def __init__(self, stream): + self.name = None + self.stream = None + self.stream_pointer = 0 + self.eof = True + self.buffer = '' + self.pointer = 0 + self.raw_buffer = None + self.raw_decode = None + self.encoding = None + self.index = 0 + self.line = 0 + self.column = 0 + if isinstance(stream, str): + self.name = "" + self.check_printable(stream) + self.buffer = stream+'\0' + elif isinstance(stream, bytes): + self.name = "" + self.raw_buffer = stream + self.determine_encoding() + else: + self.stream = stream + self.name = getattr(stream, 'name', "") + self.eof = False + self.raw_buffer = None + self.determine_encoding() + + def peek(self, index=0): + try: + return self.buffer[self.pointer+index] + except IndexError: + self.update(index+1) + return self.buffer[self.pointer+index] + + def prefix(self, length=1): + if self.pointer+length >= len(self.buffer): + self.update(length) + return self.buffer[self.pointer:self.pointer+length] + + def forward(self, length=1): + if self.pointer+length+1 >= len(self.buffer): + self.update(length+1) + while length: + ch = self.buffer[self.pointer] + self.pointer += 1 + self.index += 1 + if ch in '\n\x85\u2028\u2029' \ + or (ch == '\r' and self.buffer[self.pointer] != '\n'): + self.line += 1 + self.column = 0 + elif ch != '\uFEFF': + self.column += 1 + length -= 1 + + def get_mark(self): + if self.stream is None: + return Mark(self.name, self.index, self.line, self.column, + self.buffer, self.pointer) + else: + return Mark(self.name, self.index, self.line, self.column, + None, None) + + def determine_encoding(self): + while not self.eof and (self.raw_buffer is None or len(self.raw_buffer) < 2): + self.update_raw() + if isinstance(self.raw_buffer, bytes): + if self.raw_buffer.startswith(codecs.BOM_UTF16_LE): + self.raw_decode = codecs.utf_16_le_decode + self.encoding = 'utf-16-le' + elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE): + self.raw_decode = codecs.utf_16_be_decode + self.encoding = 'utf-16-be' + else: + self.raw_decode = codecs.utf_8_decode + self.encoding = 'utf-8' + self.update(1) + + NON_PRINTABLE = re.compile('[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]') + def check_printable(self, data): + match = self.NON_PRINTABLE.search(data) + if match: + character = match.group() + position = self.index+(len(self.buffer)-self.pointer)+match.start() + raise ReaderError(self.name, position, ord(character), + 'unicode', "special characters are not allowed") + + def update(self, length): + if self.raw_buffer is None: + return + self.buffer = self.buffer[self.pointer:] + self.pointer = 0 + while len(self.buffer) < length: + if not self.eof: + self.update_raw() + if self.raw_decode is not None: + try: + data, converted = self.raw_decode(self.raw_buffer, + 'strict', self.eof) + except UnicodeDecodeError as exc: + character = self.raw_buffer[exc.start] + if self.stream is not None: + position = self.stream_pointer-len(self.raw_buffer)+exc.start + else: + position = exc.start + raise ReaderError(self.name, position, character, + exc.encoding, exc.reason) + else: + data = self.raw_buffer + converted = len(data) + self.check_printable(data) + self.buffer += data + self.raw_buffer = self.raw_buffer[converted:] + if self.eof: + self.buffer += '\0' + self.raw_buffer = None + break + + def update_raw(self, size=4096): + data = self.stream.read(size) + if self.raw_buffer is None: + self.raw_buffer = data + else: + self.raw_buffer += data + self.stream_pointer += len(data) + if not data: + self.eof = True diff --git a/pipenv/patched/yaml3/representer.py b/pipenv/patched/yaml3/representer.py new file mode 100644 index 00000000..3b0b192e --- /dev/null +++ b/pipenv/patched/yaml3/representer.py @@ -0,0 +1,389 @@ + +__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer', + 'RepresenterError'] + +from .error import * +from .nodes import * + +import datetime, copyreg, types, base64, collections + +class RepresenterError(YAMLError): + pass + +class BaseRepresenter: + + yaml_representers = {} + yaml_multi_representers = {} + + def __init__(self, default_style=None, default_flow_style=False, sort_keys=True): + self.default_style = default_style + self.sort_keys = sort_keys + self.default_flow_style = default_flow_style + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + + def represent(self, data): + node = self.represent_data(data) + self.serialize(node) + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + + def represent_data(self, data): + if self.ignore_aliases(data): + self.alias_key = None + else: + self.alias_key = id(data) + if self.alias_key is not None: + if self.alias_key in self.represented_objects: + node = self.represented_objects[self.alias_key] + #if node is None: + # raise RepresenterError("recursive objects are not allowed: %r" % data) + return node + #self.represented_objects[alias_key] = None + self.object_keeper.append(data) + data_types = type(data).__mro__ + if data_types[0] in self.yaml_representers: + node = self.yaml_representers[data_types[0]](self, data) + else: + for data_type in data_types: + if data_type in self.yaml_multi_representers: + node = self.yaml_multi_representers[data_type](self, data) + break + else: + if None in self.yaml_multi_representers: + node = self.yaml_multi_representers[None](self, data) + elif None in self.yaml_representers: + node = self.yaml_representers[None](self, data) + else: + node = ScalarNode(None, str(data)) + #if alias_key is not None: + # self.represented_objects[alias_key] = node + return node + + @classmethod + def add_representer(cls, data_type, representer): + if not 'yaml_representers' in cls.__dict__: + cls.yaml_representers = cls.yaml_representers.copy() + cls.yaml_representers[data_type] = representer + + @classmethod + def add_multi_representer(cls, data_type, representer): + if not 'yaml_multi_representers' in cls.__dict__: + cls.yaml_multi_representers = cls.yaml_multi_representers.copy() + cls.yaml_multi_representers[data_type] = representer + + def represent_scalar(self, tag, value, style=None): + if style is None: + style = self.default_style + node = ScalarNode(tag, value, style=style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + return node + + def represent_sequence(self, tag, sequence, flow_style=None): + value = [] + node = SequenceNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + for item in sequence: + node_item = self.represent_data(item) + if not (isinstance(node_item, ScalarNode) and not node_item.style): + best_style = False + value.append(node_item) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def represent_mapping(self, tag, mapping, flow_style=None): + value = [] + node = MappingNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + if hasattr(mapping, 'items'): + mapping = list(mapping.items()) + if self.sort_keys: + try: + mapping = sorted(mapping) + except TypeError: + pass + for item_key, item_value in mapping: + node_key = self.represent_data(item_key) + node_value = self.represent_data(item_value) + if not (isinstance(node_key, ScalarNode) and not node_key.style): + best_style = False + if not (isinstance(node_value, ScalarNode) and not node_value.style): + best_style = False + value.append((node_key, node_value)) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def ignore_aliases(self, data): + return False + +class SafeRepresenter(BaseRepresenter): + + def ignore_aliases(self, data): + if data is None: + return True + if isinstance(data, tuple) and data == (): + return True + if isinstance(data, (str, bytes, bool, int, float)): + return True + + def represent_none(self, data): + return self.represent_scalar('tag:yaml.org,2002:null', 'null') + + def represent_str(self, data): + return self.represent_scalar('tag:yaml.org,2002:str', data) + + def represent_binary(self, data): + if hasattr(base64, 'encodebytes'): + data = base64.encodebytes(data).decode('ascii') + else: + data = base64.encodestring(data).decode('ascii') + return self.represent_scalar('tag:yaml.org,2002:binary', data, style='|') + + def represent_bool(self, data): + if data: + value = 'true' + else: + value = 'false' + return self.represent_scalar('tag:yaml.org,2002:bool', value) + + def represent_int(self, data): + return self.represent_scalar('tag:yaml.org,2002:int', str(data)) + + inf_value = 1e300 + while repr(inf_value) != repr(inf_value*inf_value): + inf_value *= inf_value + + def represent_float(self, data): + if data != data or (data == 0.0 and data == 1.0): + value = '.nan' + elif data == self.inf_value: + value = '.inf' + elif data == -self.inf_value: + value = '-.inf' + else: + value = repr(data).lower() + # Note that in some cases `repr(data)` represents a float number + # without the decimal parts. For instance: + # >>> repr(1e17) + # '1e17' + # Unfortunately, this is not a valid float representation according + # to the definition of the `!!float` tag. We fix this by adding + # '.0' before the 'e' symbol. + if '.' not in value and 'e' in value: + value = value.replace('e', '.0e', 1) + return self.represent_scalar('tag:yaml.org,2002:float', value) + + def represent_list(self, data): + #pairs = (len(data) > 0 and isinstance(data, list)) + #if pairs: + # for item in data: + # if not isinstance(item, tuple) or len(item) != 2: + # pairs = False + # break + #if not pairs: + return self.represent_sequence('tag:yaml.org,2002:seq', data) + #value = [] + #for item_key, item_value in data: + # value.append(self.represent_mapping(u'tag:yaml.org,2002:map', + # [(item_key, item_value)])) + #return SequenceNode(u'tag:yaml.org,2002:pairs', value) + + def represent_dict(self, data): + return self.represent_mapping('tag:yaml.org,2002:map', data) + + def represent_set(self, data): + value = {} + for key in data: + value[key] = None + return self.represent_mapping('tag:yaml.org,2002:set', value) + + def represent_date(self, data): + value = data.isoformat() + return self.represent_scalar('tag:yaml.org,2002:timestamp', value) + + def represent_datetime(self, data): + value = data.isoformat(' ') + return self.represent_scalar('tag:yaml.org,2002:timestamp', value) + + def represent_yaml_object(self, tag, data, cls, flow_style=None): + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__.copy() + return self.represent_mapping(tag, state, flow_style=flow_style) + + def represent_undefined(self, data): + raise RepresenterError("cannot represent an object", data) + +SafeRepresenter.add_representer(type(None), + SafeRepresenter.represent_none) + +SafeRepresenter.add_representer(str, + SafeRepresenter.represent_str) + +SafeRepresenter.add_representer(bytes, + SafeRepresenter.represent_binary) + +SafeRepresenter.add_representer(bool, + SafeRepresenter.represent_bool) + +SafeRepresenter.add_representer(int, + SafeRepresenter.represent_int) + +SafeRepresenter.add_representer(float, + SafeRepresenter.represent_float) + +SafeRepresenter.add_representer(list, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(tuple, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(dict, + SafeRepresenter.represent_dict) + +SafeRepresenter.add_representer(set, + SafeRepresenter.represent_set) + +SafeRepresenter.add_representer(datetime.date, + SafeRepresenter.represent_date) + +SafeRepresenter.add_representer(datetime.datetime, + SafeRepresenter.represent_datetime) + +SafeRepresenter.add_representer(None, + SafeRepresenter.represent_undefined) + +class Representer(SafeRepresenter): + + def represent_complex(self, data): + if data.imag == 0.0: + data = '%r' % data.real + elif data.real == 0.0: + data = '%rj' % data.imag + elif data.imag > 0: + data = '%r+%rj' % (data.real, data.imag) + else: + data = '%r%rj' % (data.real, data.imag) + return self.represent_scalar('tag:yaml.org,2002:python/complex', data) + + def represent_tuple(self, data): + return self.represent_sequence('tag:yaml.org,2002:python/tuple', data) + + def represent_name(self, data): + name = '%s.%s' % (data.__module__, data.__name__) + return self.represent_scalar('tag:yaml.org,2002:python/name:'+name, '') + + def represent_module(self, data): + return self.represent_scalar( + 'tag:yaml.org,2002:python/module:'+data.__name__, '') + + def represent_object(self, data): + # We use __reduce__ API to save the data. data.__reduce__ returns + # a tuple of length 2-5: + # (function, args, state, listitems, dictitems) + + # For reconstructing, we calls function(*args), then set its state, + # listitems, and dictitems if they are not None. + + # A special case is when function.__name__ == '__newobj__'. In this + # case we create the object with args[0].__new__(*args). + + # Another special case is when __reduce__ returns a string - we don't + # support it. + + # We produce a !!python/object, !!python/object/new or + # !!python/object/apply node. + + cls = type(data) + if cls in copyreg.dispatch_table: + reduce = copyreg.dispatch_table[cls](data) + elif hasattr(data, '__reduce_ex__'): + reduce = data.__reduce_ex__(2) + elif hasattr(data, '__reduce__'): + reduce = data.__reduce__() + else: + raise RepresenterError("cannot represent an object", data) + reduce = (list(reduce)+[None]*5)[:5] + function, args, state, listitems, dictitems = reduce + args = list(args) + if state is None: + state = {} + if listitems is not None: + listitems = list(listitems) + if dictitems is not None: + dictitems = dict(dictitems) + if function.__name__ == '__newobj__': + function = args[0] + args = args[1:] + tag = 'tag:yaml.org,2002:python/object/new:' + newobj = True + else: + tag = 'tag:yaml.org,2002:python/object/apply:' + newobj = False + function_name = '%s.%s' % (function.__module__, function.__name__) + if not args and not listitems and not dictitems \ + and isinstance(state, dict) and newobj: + return self.represent_mapping( + 'tag:yaml.org,2002:python/object:'+function_name, state) + if not listitems and not dictitems \ + and isinstance(state, dict) and not state: + return self.represent_sequence(tag+function_name, args) + value = {} + if args: + value['args'] = args + if state or not isinstance(state, dict): + value['state'] = state + if listitems: + value['listitems'] = listitems + if dictitems: + value['dictitems'] = dictitems + return self.represent_mapping(tag+function_name, value) + + def represent_ordered_dict(self, data): + # Provide uniform representation across different Python versions. + data_type = type(data) + tag = 'tag:yaml.org,2002:python/object/apply:%s.%s' \ + % (data_type.__module__, data_type.__name__) + items = [[key, value] for key, value in data.items()] + return self.represent_sequence(tag, [items]) + +Representer.add_representer(complex, + Representer.represent_complex) + +Representer.add_representer(tuple, + Representer.represent_tuple) + +Representer.add_representer(type, + Representer.represent_name) + +Representer.add_representer(collections.OrderedDict, + Representer.represent_ordered_dict) + +Representer.add_representer(types.FunctionType, + Representer.represent_name) + +Representer.add_representer(types.BuiltinFunctionType, + Representer.represent_name) + +Representer.add_representer(types.ModuleType, + Representer.represent_module) + +Representer.add_multi_representer(object, + Representer.represent_object) + diff --git a/pipenv/patched/yaml3/resolver.py b/pipenv/patched/yaml3/resolver.py new file mode 100644 index 00000000..02b82e73 --- /dev/null +++ b/pipenv/patched/yaml3/resolver.py @@ -0,0 +1,227 @@ + +__all__ = ['BaseResolver', 'Resolver'] + +from .error import * +from .nodes import * + +import re + +class ResolverError(YAMLError): + pass + +class BaseResolver: + + DEFAULT_SCALAR_TAG = 'tag:yaml.org,2002:str' + DEFAULT_SEQUENCE_TAG = 'tag:yaml.org,2002:seq' + DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map' + + yaml_implicit_resolvers = {} + yaml_path_resolvers = {} + + def __init__(self): + self.resolver_exact_paths = [] + self.resolver_prefix_paths = [] + + @classmethod + def add_implicit_resolver(cls, tag, regexp, first): + if not 'yaml_implicit_resolvers' in cls.__dict__: + implicit_resolvers = {} + for key in cls.yaml_implicit_resolvers: + implicit_resolvers[key] = cls.yaml_implicit_resolvers[key][:] + cls.yaml_implicit_resolvers = implicit_resolvers + if first is None: + first = [None] + for ch in first: + cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp)) + + @classmethod + def add_path_resolver(cls, tag, path, kind=None): + # Note: `add_path_resolver` is experimental. The API could be changed. + # `new_path` is a pattern that is matched against the path from the + # root to the node that is being considered. `node_path` elements are + # tuples `(node_check, index_check)`. `node_check` is a node class: + # `ScalarNode`, `SequenceNode`, `MappingNode` or `None`. `None` + # matches any kind of a node. `index_check` could be `None`, a boolean + # value, a string value, or a number. `None` and `False` match against + # any _value_ of sequence and mapping nodes. `True` matches against + # any _key_ of a mapping node. A string `index_check` matches against + # a mapping value that corresponds to a scalar key which content is + # equal to the `index_check` value. An integer `index_check` matches + # against a sequence value with the index equal to `index_check`. + if not 'yaml_path_resolvers' in cls.__dict__: + cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy() + new_path = [] + for element in path: + if isinstance(element, (list, tuple)): + if len(element) == 2: + node_check, index_check = element + elif len(element) == 1: + node_check = element[0] + index_check = True + else: + raise ResolverError("Invalid path element: %s" % element) + else: + node_check = None + index_check = element + if node_check is str: + node_check = ScalarNode + elif node_check is list: + node_check = SequenceNode + elif node_check is dict: + node_check = MappingNode + elif node_check not in [ScalarNode, SequenceNode, MappingNode] \ + and not isinstance(node_check, str) \ + and node_check is not None: + raise ResolverError("Invalid node checker: %s" % node_check) + if not isinstance(index_check, (str, int)) \ + and index_check is not None: + raise ResolverError("Invalid index checker: %s" % index_check) + new_path.append((node_check, index_check)) + if kind is str: + kind = ScalarNode + elif kind is list: + kind = SequenceNode + elif kind is dict: + kind = MappingNode + elif kind not in [ScalarNode, SequenceNode, MappingNode] \ + and kind is not None: + raise ResolverError("Invalid node kind: %s" % kind) + cls.yaml_path_resolvers[tuple(new_path), kind] = tag + + def descend_resolver(self, current_node, current_index): + if not self.yaml_path_resolvers: + return + exact_paths = {} + prefix_paths = [] + if current_node: + depth = len(self.resolver_prefix_paths) + for path, kind in self.resolver_prefix_paths[-1]: + if self.check_resolver_prefix(depth, path, kind, + current_node, current_index): + if len(path) > depth: + prefix_paths.append((path, kind)) + else: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + for path, kind in self.yaml_path_resolvers: + if not path: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + prefix_paths.append((path, kind)) + self.resolver_exact_paths.append(exact_paths) + self.resolver_prefix_paths.append(prefix_paths) + + def ascend_resolver(self): + if not self.yaml_path_resolvers: + return + self.resolver_exact_paths.pop() + self.resolver_prefix_paths.pop() + + def check_resolver_prefix(self, depth, path, kind, + current_node, current_index): + node_check, index_check = path[depth-1] + if isinstance(node_check, str): + if current_node.tag != node_check: + return + elif node_check is not None: + if not isinstance(current_node, node_check): + return + if index_check is True and current_index is not None: + return + if (index_check is False or index_check is None) \ + and current_index is None: + return + if isinstance(index_check, str): + if not (isinstance(current_index, ScalarNode) + and index_check == current_index.value): + return + elif isinstance(index_check, int) and not isinstance(index_check, bool): + if index_check != current_index: + return + return True + + def resolve(self, kind, value, implicit): + if kind is ScalarNode and implicit[0]: + if value == '': + resolvers = self.yaml_implicit_resolvers.get('', []) + else: + resolvers = self.yaml_implicit_resolvers.get(value[0], []) + resolvers += self.yaml_implicit_resolvers.get(None, []) + for tag, regexp in resolvers: + if regexp.match(value): + return tag + implicit = implicit[1] + if self.yaml_path_resolvers: + exact_paths = self.resolver_exact_paths[-1] + if kind in exact_paths: + return exact_paths[kind] + if None in exact_paths: + return exact_paths[None] + if kind is ScalarNode: + return self.DEFAULT_SCALAR_TAG + elif kind is SequenceNode: + return self.DEFAULT_SEQUENCE_TAG + elif kind is MappingNode: + return self.DEFAULT_MAPPING_TAG + +class Resolver(BaseResolver): + pass + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:bool', + re.compile(r'''^(?:yes|Yes|YES|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF)$''', re.X), + list('yYnNtTfFoO')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:float', + re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? + |\.[0-9_]+(?:[eE][-+][0-9]+)? + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]* + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$''', re.X), + list('-+0123456789.')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:int', + re.compile(r'''^(?:[-+]?0b[0-1_]+ + |[-+]?0[0-7_]+ + |[-+]?(?:0|[1-9][0-9_]*) + |[-+]?0x[0-9a-fA-F_]+ + |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), + list('-+0123456789')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:merge', + re.compile(r'^(?:<<)$'), + ['<']) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:null', + re.compile(r'''^(?: ~ + |null|Null|NULL + | )$''', re.X), + ['~', 'n', 'N', '']) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:timestamp', + re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] + |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? + (?:[Tt]|[ \t]+)[0-9][0-9]? + :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)? + (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), + list('0123456789')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:value', + re.compile(r'^(?:=)$'), + ['=']) + +# The following resolver is only for documentation purposes. It cannot work +# because plain scalars cannot start with '!', '&', or '*'. +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:yaml', + re.compile(r'^(?:!|&|\*)$'), + list('!&*')) + diff --git a/pipenv/patched/yaml3/scanner.py b/pipenv/patched/yaml3/scanner.py new file mode 100644 index 00000000..7437ede1 --- /dev/null +++ b/pipenv/patched/yaml3/scanner.py @@ -0,0 +1,1435 @@ + +# Scanner produces tokens of the following types: +# STREAM-START +# STREAM-END +# DIRECTIVE(name, value) +# DOCUMENT-START +# DOCUMENT-END +# BLOCK-SEQUENCE-START +# BLOCK-MAPPING-START +# BLOCK-END +# FLOW-SEQUENCE-START +# FLOW-MAPPING-START +# FLOW-SEQUENCE-END +# FLOW-MAPPING-END +# BLOCK-ENTRY +# FLOW-ENTRY +# KEY +# VALUE +# ALIAS(value) +# ANCHOR(value) +# TAG(value) +# SCALAR(value, plain, style) +# +# Read comments in the Scanner code for more details. +# + +__all__ = ['Scanner', 'ScannerError'] + +from .error import MarkedYAMLError +from .tokens import * + +class ScannerError(MarkedYAMLError): + pass + +class SimpleKey: + # See below simple keys treatment. + + def __init__(self, token_number, required, index, line, column, mark): + self.token_number = token_number + self.required = required + self.index = index + self.line = line + self.column = column + self.mark = mark + +class Scanner: + + def __init__(self): + """Initialize the scanner.""" + # It is assumed that Scanner and Reader will have a common descendant. + # Reader do the dirty work of checking for BOM and converting the + # input data to Unicode. It also adds NUL to the end. + # + # Reader supports the following methods + # self.peek(i=0) # peek the next i-th character + # self.prefix(l=1) # peek the next l characters + # self.forward(l=1) # read the next l characters and move the pointer. + + # Had we reached the end of the stream? + self.done = False + + # The number of unclosed '{' and '['. `flow_level == 0` means block + # context. + self.flow_level = 0 + + # List of processed tokens that are not yet emitted. + self.tokens = [] + + # Add the STREAM-START token. + self.fetch_stream_start() + + # Number of tokens that were emitted through the `get_token` method. + self.tokens_taken = 0 + + # The current indentation level. + self.indent = -1 + + # Past indentation levels. + self.indents = [] + + # Variables related to simple keys treatment. + + # A simple key is a key that is not denoted by the '?' indicator. + # Example of simple keys: + # --- + # block simple key: value + # ? not a simple key: + # : { flow simple key: value } + # We emit the KEY token before all keys, so when we find a potential + # simple key, we try to locate the corresponding ':' indicator. + # Simple keys should be limited to a single line and 1024 characters. + + # Can a simple key start at the current position? A simple key may + # start: + # - at the beginning of the line, not counting indentation spaces + # (in block context), + # - after '{', '[', ',' (in the flow context), + # - after '?', ':', '-' (in the block context). + # In the block context, this flag also signifies if a block collection + # may start at the current position. + self.allow_simple_key = True + + # Keep track of possible simple keys. This is a dictionary. The key + # is `flow_level`; there can be no more that one possible simple key + # for each level. The value is a SimpleKey record: + # (token_number, required, index, line, column, mark) + # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow), + # '[', or '{' tokens. + self.possible_simple_keys = {} + + # Public methods. + + def check_token(self, *choices): + # Check if the next token is one of the given types. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + if not choices: + return True + for choice in choices: + if isinstance(self.tokens[0], choice): + return True + return False + + def peek_token(self): + # Return the next token, but do not delete if from the queue. + # Return None if no more tokens. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + return self.tokens[0] + else: + return None + + def get_token(self): + # Return the next token. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + self.tokens_taken += 1 + return self.tokens.pop(0) + + # Private methods. + + def need_more_tokens(self): + if self.done: + return False + if not self.tokens: + return True + # The current token may be a potential simple key, so we + # need to look further. + self.stale_possible_simple_keys() + if self.next_possible_simple_key() == self.tokens_taken: + return True + + def fetch_more_tokens(self): + + # Eat whitespaces and comments until we reach the next token. + self.scan_to_next_token() + + # Remove obsolete possible simple keys. + self.stale_possible_simple_keys() + + # Compare the current indentation and column. It may add some tokens + # and decrease the current indentation level. + self.unwind_indent(self.column) + + # Peek the next character. + ch = self.peek() + + # Is it the end of stream? + if ch == '\0': + return self.fetch_stream_end() + + # Is it a directive? + if ch == '%' and self.check_directive(): + return self.fetch_directive() + + # Is it the document start? + if ch == '-' and self.check_document_start(): + return self.fetch_document_start() + + # Is it the document end? + if ch == '.' and self.check_document_end(): + return self.fetch_document_end() + + # TODO: support for BOM within a stream. + #if ch == '\uFEFF': + # return self.fetch_bom() <-- issue BOMToken + + # Note: the order of the following checks is NOT significant. + + # Is it the flow sequence start indicator? + if ch == '[': + return self.fetch_flow_sequence_start() + + # Is it the flow mapping start indicator? + if ch == '{': + return self.fetch_flow_mapping_start() + + # Is it the flow sequence end indicator? + if ch == ']': + return self.fetch_flow_sequence_end() + + # Is it the flow mapping end indicator? + if ch == '}': + return self.fetch_flow_mapping_end() + + # Is it the flow entry indicator? + if ch == ',': + return self.fetch_flow_entry() + + # Is it the block entry indicator? + if ch == '-' and self.check_block_entry(): + return self.fetch_block_entry() + + # Is it the key indicator? + if ch == '?' and self.check_key(): + return self.fetch_key() + + # Is it the value indicator? + if ch == ':' and self.check_value(): + return self.fetch_value() + + # Is it an alias? + if ch == '*': + return self.fetch_alias() + + # Is it an anchor? + if ch == '&': + return self.fetch_anchor() + + # Is it a tag? + if ch == '!': + return self.fetch_tag() + + # Is it a literal scalar? + if ch == '|' and not self.flow_level: + return self.fetch_literal() + + # Is it a folded scalar? + if ch == '>' and not self.flow_level: + return self.fetch_folded() + + # Is it a single quoted scalar? + if ch == '\'': + return self.fetch_single() + + # Is it a double quoted scalar? + if ch == '\"': + return self.fetch_double() + + # It must be a plain scalar then. + if self.check_plain(): + return self.fetch_plain() + + # No? It's an error. Let's produce a nice error message. + raise ScannerError("while scanning for the next token", None, + "found character %r that cannot start any token" % ch, + self.get_mark()) + + # Simple keys treatment. + + def next_possible_simple_key(self): + # Return the number of the nearest possible simple key. Actually we + # don't need to loop through the whole dictionary. We may replace it + # with the following code: + # if not self.possible_simple_keys: + # return None + # return self.possible_simple_keys[ + # min(self.possible_simple_keys.keys())].token_number + min_token_number = None + for level in self.possible_simple_keys: + key = self.possible_simple_keys[level] + if min_token_number is None or key.token_number < min_token_number: + min_token_number = key.token_number + return min_token_number + + def stale_possible_simple_keys(self): + # Remove entries that are no longer possible simple keys. According to + # the YAML specification, simple keys + # - should be limited to a single line, + # - should be no longer than 1024 characters. + # Disabling this procedure will allow simple keys of any length and + # height (may cause problems if indentation is broken though). + for level in list(self.possible_simple_keys): + key = self.possible_simple_keys[level] + if key.line != self.line \ + or self.index-key.index > 1024: + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self.get_mark()) + del self.possible_simple_keys[level] + + def save_possible_simple_key(self): + # The next token may start a simple key. We check if it's possible + # and save its position. This function is called for + # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'. + + # Check if a simple key is required at the current position. + required = not self.flow_level and self.indent == self.column + + # The next token might be a simple key. Let's save it's number and + # position. + if self.allow_simple_key: + self.remove_possible_simple_key() + token_number = self.tokens_taken+len(self.tokens) + key = SimpleKey(token_number, required, + self.index, self.line, self.column, self.get_mark()) + self.possible_simple_keys[self.flow_level] = key + + def remove_possible_simple_key(self): + # Remove the saved possible key position at the current flow level. + if self.flow_level in self.possible_simple_keys: + key = self.possible_simple_keys[self.flow_level] + + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self.get_mark()) + + del self.possible_simple_keys[self.flow_level] + + # Indentation functions. + + def unwind_indent(self, column): + + ## In flow context, tokens should respect indentation. + ## Actually the condition should be `self.indent >= column` according to + ## the spec. But this condition will prohibit intuitively correct + ## constructions such as + ## key : { + ## } + #if self.flow_level and self.indent > column: + # raise ScannerError(None, None, + # "invalid indentation or unclosed '[' or '{'", + # self.get_mark()) + + # In the flow context, indentation is ignored. We make the scanner less + # restrictive then specification requires. + if self.flow_level: + return + + # In block context, we may need to issue the BLOCK-END tokens. + while self.indent > column: + mark = self.get_mark() + self.indent = self.indents.pop() + self.tokens.append(BlockEndToken(mark, mark)) + + def add_indent(self, column): + # Check if we need to increase indentation. + if self.indent < column: + self.indents.append(self.indent) + self.indent = column + return True + return False + + # Fetchers. + + def fetch_stream_start(self): + # We always add STREAM-START as the first token and STREAM-END as the + # last token. + + # Read the token. + mark = self.get_mark() + + # Add STREAM-START. + self.tokens.append(StreamStartToken(mark, mark, + encoding=self.encoding)) + + + def fetch_stream_end(self): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. + self.remove_possible_simple_key() + self.allow_simple_key = False + self.possible_simple_keys = {} + + # Read the token. + mark = self.get_mark() + + # Add STREAM-END. + self.tokens.append(StreamEndToken(mark, mark)) + + # The steam is finished. + self.done = True + + def fetch_directive(self): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Scan and add DIRECTIVE. + self.tokens.append(self.scan_directive()) + + def fetch_document_start(self): + self.fetch_document_indicator(DocumentStartToken) + + def fetch_document_end(self): + self.fetch_document_indicator(DocumentEndToken) + + def fetch_document_indicator(self, TokenClass): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. Note that there could not be a block collection + # after '---'. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Add DOCUMENT-START or DOCUMENT-END. + start_mark = self.get_mark() + self.forward(3) + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_start(self): + self.fetch_flow_collection_start(FlowSequenceStartToken) + + def fetch_flow_mapping_start(self): + self.fetch_flow_collection_start(FlowMappingStartToken) + + def fetch_flow_collection_start(self, TokenClass): + + # '[' and '{' may start a simple key. + self.save_possible_simple_key() + + # Increase the flow level. + self.flow_level += 1 + + # Simple keys are allowed after '[' and '{'. + self.allow_simple_key = True + + # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_end(self): + self.fetch_flow_collection_end(FlowSequenceEndToken) + + def fetch_flow_mapping_end(self): + self.fetch_flow_collection_end(FlowMappingEndToken) + + def fetch_flow_collection_end(self, TokenClass): + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Decrease the flow level. + self.flow_level -= 1 + + # No simple keys after ']' or '}'. + self.allow_simple_key = False + + # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_entry(self): + + # Simple keys are allowed after ','. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add FLOW-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(FlowEntryToken(start_mark, end_mark)) + + def fetch_block_entry(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a new entry? + if not self.allow_simple_key: + raise ScannerError(None, None, + "sequence entries are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-SEQUENCE-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockSequenceStartToken(mark, mark)) + + # It's an error for the block entry to occur in the flow context, + # but we let the parser detect this. + else: + pass + + # Simple keys are allowed after '-'. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add BLOCK-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(BlockEntryToken(start_mark, end_mark)) + + def fetch_key(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a key (not necessary a simple)? + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping keys are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-MAPPING-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockMappingStartToken(mark, mark)) + + # Simple keys are allowed after '?' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add KEY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(KeyToken(start_mark, end_mark)) + + def fetch_value(self): + + # Do we determine a simple key? + if self.flow_level in self.possible_simple_keys: + + # Add KEY. + key = self.possible_simple_keys[self.flow_level] + del self.possible_simple_keys[self.flow_level] + self.tokens.insert(key.token_number-self.tokens_taken, + KeyToken(key.mark, key.mark)) + + # If this key starts a new block mapping, we need to add + # BLOCK-MAPPING-START. + if not self.flow_level: + if self.add_indent(key.column): + self.tokens.insert(key.token_number-self.tokens_taken, + BlockMappingStartToken(key.mark, key.mark)) + + # There cannot be two simple keys one after another. + self.allow_simple_key = False + + # It must be a part of a complex key. + else: + + # Block context needs additional checks. + # (Do we really need them? They will be caught by the parser + # anyway.) + if not self.flow_level: + + # We are allowed to start a complex value if and only if + # we can start a simple key. + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping values are not allowed here", + self.get_mark()) + + # If this value starts a new block mapping, we need to add + # BLOCK-MAPPING-START. It will be detected as an error later by + # the parser. + if not self.flow_level: + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockMappingStartToken(mark, mark)) + + # Simple keys are allowed after ':' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add VALUE. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(ValueToken(start_mark, end_mark)) + + def fetch_alias(self): + + # ALIAS could be a simple key. + self.save_possible_simple_key() + + # No simple keys after ALIAS. + self.allow_simple_key = False + + # Scan and add ALIAS. + self.tokens.append(self.scan_anchor(AliasToken)) + + def fetch_anchor(self): + + # ANCHOR could start a simple key. + self.save_possible_simple_key() + + # No simple keys after ANCHOR. + self.allow_simple_key = False + + # Scan and add ANCHOR. + self.tokens.append(self.scan_anchor(AnchorToken)) + + def fetch_tag(self): + + # TAG could start a simple key. + self.save_possible_simple_key() + + # No simple keys after TAG. + self.allow_simple_key = False + + # Scan and add TAG. + self.tokens.append(self.scan_tag()) + + def fetch_literal(self): + self.fetch_block_scalar(style='|') + + def fetch_folded(self): + self.fetch_block_scalar(style='>') + + def fetch_block_scalar(self, style): + + # A simple key may follow a block scalar. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Scan and add SCALAR. + self.tokens.append(self.scan_block_scalar(style)) + + def fetch_single(self): + self.fetch_flow_scalar(style='\'') + + def fetch_double(self): + self.fetch_flow_scalar(style='"') + + def fetch_flow_scalar(self, style): + + # A flow scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after flow scalars. + self.allow_simple_key = False + + # Scan and add SCALAR. + self.tokens.append(self.scan_flow_scalar(style)) + + def fetch_plain(self): + + # A plain scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after plain scalars. But note that `scan_plain` will + # change this flag if the scan is finished at the beginning of the + # line. + self.allow_simple_key = False + + # Scan and add SCALAR. May change `allow_simple_key`. + self.tokens.append(self.scan_plain()) + + # Checkers. + + def check_directive(self): + + # DIRECTIVE: ^ '%' ... + # The '%' indicator is already checked. + if self.column == 0: + return True + + def check_document_start(self): + + # DOCUMENT-START: ^ '---' (' '|'\n') + if self.column == 0: + if self.prefix(3) == '---' \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return True + + def check_document_end(self): + + # DOCUMENT-END: ^ '...' (' '|'\n') + if self.column == 0: + if self.prefix(3) == '...' \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return True + + def check_block_entry(self): + + # BLOCK-ENTRY: '-' (' '|'\n') + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029' + + def check_key(self): + + # KEY(flow context): '?' + if self.flow_level: + return True + + # KEY(block context): '?' (' '|'\n') + else: + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029' + + def check_value(self): + + # VALUE(flow context): ':' + if self.flow_level: + return True + + # VALUE(block context): ':' (' '|'\n') + else: + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029' + + def check_plain(self): + + # A plain scalar may start with any non-space character except: + # '-', '?', ':', ',', '[', ']', '{', '}', + # '#', '&', '*', '!', '|', '>', '\'', '\"', + # '%', '@', '`'. + # + # It may also start with + # '-', '?', ':' + # if it is followed by a non-space character. + # + # Note that we limit the last rule to the block context (except the + # '-' character) because we want the flow context to be space + # independent. + ch = self.peek() + return ch not in '\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \ + or (self.peek(1) not in '\0 \t\r\n\x85\u2028\u2029' + and (ch == '-' or (not self.flow_level and ch in '?:'))) + + # Scanners. + + def scan_to_next_token(self): + # We ignore spaces, line breaks and comments. + # If we find a line break in the block context, we set the flag + # `allow_simple_key` on. + # The byte order mark is stripped if it's the first character in the + # stream. We do not yet support BOM inside the stream as the + # specification requires. Any such mark will be considered as a part + # of the document. + # + # TODO: We need to make tab handling rules more sane. A good rule is + # Tabs cannot precede tokens + # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END, + # KEY(block), VALUE(block), BLOCK-ENTRY + # So the checking code is + # if : + # self.allow_simple_keys = False + # We also need to add the check for `allow_simple_keys == True` to + # `unwind_indent` before issuing BLOCK-END. + # Scanners for block, flow, and plain scalars need to be modified. + + if self.index == 0 and self.peek() == '\uFEFF': + self.forward() + found = False + while not found: + while self.peek() == ' ': + self.forward() + if self.peek() == '#': + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + if self.scan_line_break(): + if not self.flow_level: + self.allow_simple_key = True + else: + found = True + + def scan_directive(self): + # See the specification for details. + start_mark = self.get_mark() + self.forward() + name = self.scan_directive_name(start_mark) + value = None + if name == 'YAML': + value = self.scan_yaml_directive_value(start_mark) + end_mark = self.get_mark() + elif name == 'TAG': + value = self.scan_tag_directive_value(start_mark) + end_mark = self.get_mark() + else: + end_mark = self.get_mark() + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + self.scan_directive_ignored_line(start_mark) + return DirectiveToken(name, value, start_mark, end_mark) + + def scan_directive_name(self, start_mark): + # See the specification for details. + length = 0 + ch = self.peek(length) + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + return value + + def scan_yaml_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + major = self.scan_yaml_directive_number(start_mark) + if self.peek() != '.': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or '.', but found %r" % self.peek(), + self.get_mark()) + self.forward() + minor = self.scan_yaml_directive_number(start_mark) + if self.peek() not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or ' ', but found %r" % self.peek(), + self.get_mark()) + return (major, minor) + + def scan_yaml_directive_number(self, start_mark): + # See the specification for details. + ch = self.peek() + if not ('0' <= ch <= '9'): + raise ScannerError("while scanning a directive", start_mark, + "expected a digit, but found %r" % ch, self.get_mark()) + length = 0 + while '0' <= self.peek(length) <= '9': + length += 1 + value = int(self.prefix(length)) + self.forward(length) + return value + + def scan_tag_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + handle = self.scan_tag_directive_handle(start_mark) + while self.peek() == ' ': + self.forward() + prefix = self.scan_tag_directive_prefix(start_mark) + return (handle, prefix) + + def scan_tag_directive_handle(self, start_mark): + # See the specification for details. + value = self.scan_tag_handle('directive', start_mark) + ch = self.peek() + if ch != ' ': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch, self.get_mark()) + return value + + def scan_tag_directive_prefix(self, start_mark): + # See the specification for details. + value = self.scan_tag_uri('directive', start_mark) + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch, self.get_mark()) + return value + + def scan_directive_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + if self.peek() == '#': + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in '\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a comment or a line break, but found %r" + % ch, self.get_mark()) + self.scan_line_break() + + def scan_anchor(self, TokenClass): + # The specification does not restrict characters for anchors and + # aliases. This may lead to problems, for instance, the document: + # [ *alias, value ] + # can be interpreted in two ways, as + # [ "value" ] + # and + # [ *alias , "value" ] + # Therefore we restrict aliases to numbers and ASCII letters. + start_mark = self.get_mark() + indicator = self.peek() + if indicator == '*': + name = 'alias' + else: + name = 'anchor' + self.forward() + length = 0 + ch = self.peek(length) + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in '\0 \t\r\n\x85\u2028\u2029?:,]}%@`': + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + end_mark = self.get_mark() + return TokenClass(value, start_mark, end_mark) + + def scan_tag(self): + # See the specification for details. + start_mark = self.get_mark() + ch = self.peek(1) + if ch == '<': + handle = None + self.forward(2) + suffix = self.scan_tag_uri('tag', start_mark) + if self.peek() != '>': + raise ScannerError("while parsing a tag", start_mark, + "expected '>', but found %r" % self.peek(), + self.get_mark()) + self.forward() + elif ch in '\0 \t\r\n\x85\u2028\u2029': + handle = None + suffix = '!' + self.forward() + else: + length = 1 + use_handle = False + while ch not in '\0 \r\n\x85\u2028\u2029': + if ch == '!': + use_handle = True + break + length += 1 + ch = self.peek(length) + handle = '!' + if use_handle: + handle = self.scan_tag_handle('tag', start_mark) + else: + handle = '!' + self.forward() + suffix = self.scan_tag_uri('tag', start_mark) + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a tag", start_mark, + "expected ' ', but found %r" % ch, self.get_mark()) + value = (handle, suffix) + end_mark = self.get_mark() + return TagToken(value, start_mark, end_mark) + + def scan_block_scalar(self, style): + # See the specification for details. + + if style == '>': + folded = True + else: + folded = False + + chunks = [] + start_mark = self.get_mark() + + # Scan the header. + self.forward() + chomping, increment = self.scan_block_scalar_indicators(start_mark) + self.scan_block_scalar_ignored_line(start_mark) + + # Determine the indentation level and go to the first non-empty line. + min_indent = self.indent+1 + if min_indent < 1: + min_indent = 1 + if increment is None: + breaks, max_indent, end_mark = self.scan_block_scalar_indentation() + indent = max(min_indent, max_indent) + else: + indent = min_indent+increment-1 + breaks, end_mark = self.scan_block_scalar_breaks(indent) + line_break = '' + + # Scan the inner part of the block scalar. + while self.column == indent and self.peek() != '\0': + chunks.extend(breaks) + leading_non_space = self.peek() not in ' \t' + length = 0 + while self.peek(length) not in '\0\r\n\x85\u2028\u2029': + length += 1 + chunks.append(self.prefix(length)) + self.forward(length) + line_break = self.scan_line_break() + breaks, end_mark = self.scan_block_scalar_breaks(indent) + if self.column == indent and self.peek() != '\0': + + # Unfortunately, folding rules are ambiguous. + # + # This is the folding according to the specification: + + if folded and line_break == '\n' \ + and leading_non_space and self.peek() not in ' \t': + if not breaks: + chunks.append(' ') + else: + chunks.append(line_break) + + # This is Clark Evans's interpretation (also in the spec + # examples): + # + #if folded and line_break == '\n': + # if not breaks: + # if self.peek() not in ' \t': + # chunks.append(' ') + # else: + # chunks.append(line_break) + #else: + # chunks.append(line_break) + else: + break + + # Chomp the tail. + if chomping is not False: + chunks.append(line_break) + if chomping is True: + chunks.extend(breaks) + + # We are done. + return ScalarToken(''.join(chunks), False, start_mark, end_mark, + style) + + def scan_block_scalar_indicators(self, start_mark): + # See the specification for details. + chomping = None + increment = None + ch = self.peek() + if ch in '+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch in '0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + elif ch in '0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + ch = self.peek() + if ch in '+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected chomping or indentation indicators, but found %r" + % ch, self.get_mark()) + return chomping, increment + + def scan_block_scalar_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + if self.peek() == '#': + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in '\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected a comment or a line break, but found %r" % ch, + self.get_mark()) + self.scan_line_break() + + def scan_block_scalar_indentation(self): + # See the specification for details. + chunks = [] + max_indent = 0 + end_mark = self.get_mark() + while self.peek() in ' \r\n\x85\u2028\u2029': + if self.peek() != ' ': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + else: + self.forward() + if self.column > max_indent: + max_indent = self.column + return chunks, max_indent, end_mark + + def scan_block_scalar_breaks(self, indent): + # See the specification for details. + chunks = [] + end_mark = self.get_mark() + while self.column < indent and self.peek() == ' ': + self.forward() + while self.peek() in '\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + while self.column < indent and self.peek() == ' ': + self.forward() + return chunks, end_mark + + def scan_flow_scalar(self, style): + # See the specification for details. + # Note that we loose indentation rules for quoted scalars. Quoted + # scalars don't need to adhere indentation because " and ' clearly + # mark the beginning and the end of them. Therefore we are less + # restrictive then the specification requires. We only need to check + # that document separators are not included in scalars. + if style == '"': + double = True + else: + double = False + chunks = [] + start_mark = self.get_mark() + quote = self.peek() + self.forward() + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + while self.peek() != quote: + chunks.extend(self.scan_flow_scalar_spaces(double, start_mark)) + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + self.forward() + end_mark = self.get_mark() + return ScalarToken(''.join(chunks), False, start_mark, end_mark, + style) + + ESCAPE_REPLACEMENTS = { + '0': '\0', + 'a': '\x07', + 'b': '\x08', + 't': '\x09', + '\t': '\x09', + 'n': '\x0A', + 'v': '\x0B', + 'f': '\x0C', + 'r': '\x0D', + 'e': '\x1B', + ' ': '\x20', + '\"': '\"', + '\\': '\\', + '/': '/', + 'N': '\x85', + '_': '\xA0', + 'L': '\u2028', + 'P': '\u2029', + } + + ESCAPE_CODES = { + 'x': 2, + 'u': 4, + 'U': 8, + } + + def scan_flow_scalar_non_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + length = 0 + while self.peek(length) not in '\'\"\\\0 \t\r\n\x85\u2028\u2029': + length += 1 + if length: + chunks.append(self.prefix(length)) + self.forward(length) + ch = self.peek() + if not double and ch == '\'' and self.peek(1) == '\'': + chunks.append('\'') + self.forward(2) + elif (double and ch == '\'') or (not double and ch in '\"\\'): + chunks.append(ch) + self.forward() + elif double and ch == '\\': + self.forward() + ch = self.peek() + if ch in self.ESCAPE_REPLACEMENTS: + chunks.append(self.ESCAPE_REPLACEMENTS[ch]) + self.forward() + elif ch in self.ESCAPE_CODES: + length = self.ESCAPE_CODES[ch] + self.forward() + for k in range(length): + if self.peek(k) not in '0123456789ABCDEFabcdef': + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "expected escape sequence of %d hexdecimal numbers, but found %r" % + (length, self.peek(k)), self.get_mark()) + code = int(self.prefix(length), 16) + chunks.append(chr(code)) + self.forward(length) + elif ch in '\r\n\x85\u2028\u2029': + self.scan_line_break() + chunks.extend(self.scan_flow_scalar_breaks(double, start_mark)) + else: + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "found unknown escape character %r" % ch, self.get_mark()) + else: + return chunks + + def scan_flow_scalar_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + length = 0 + while self.peek(length) in ' \t': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch == '\0': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected end of stream", self.get_mark()) + elif ch in '\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + breaks = self.scan_flow_scalar_breaks(double, start_mark) + if line_break != '\n': + chunks.append(line_break) + elif not breaks: + chunks.append(' ') + chunks.extend(breaks) + else: + chunks.append(whitespaces) + return chunks + + def scan_flow_scalar_breaks(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + # Instead of checking indentation, we check for document + # separators. + prefix = self.prefix(3) + if (prefix == '---' or prefix == '...') \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected document separator", self.get_mark()) + while self.peek() in ' \t': + self.forward() + if self.peek() in '\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + else: + return chunks + + def scan_plain(self): + # See the specification for details. + # We add an additional restriction for the flow context: + # plain scalars in the flow context cannot contain ',' or '?'. + # We also keep track of the `allow_simple_key` flag here. + # Indentation rules are loosed for the flow context. + chunks = [] + start_mark = self.get_mark() + end_mark = start_mark + indent = self.indent+1 + # We allow zero indentation for scalars, but then we need to check for + # document separators at the beginning of the line. + #if indent == 0: + # indent = 1 + spaces = [] + while True: + length = 0 + if self.peek() == '#': + break + while True: + ch = self.peek(length) + if ch in '\0 \t\r\n\x85\u2028\u2029' \ + or (ch == ':' and + self.peek(length+1) in '\0 \t\r\n\x85\u2028\u2029' + + (u',[]{}' if self.flow_level else u''))\ + or (self.flow_level and ch in ',?[]{}'): + break + length += 1 + if length == 0: + break + self.allow_simple_key = False + chunks.extend(spaces) + chunks.append(self.prefix(length)) + self.forward(length) + end_mark = self.get_mark() + spaces = self.scan_plain_spaces(indent, start_mark) + if not spaces or self.peek() == '#' \ + or (not self.flow_level and self.column < indent): + break + return ScalarToken(''.join(chunks), True, start_mark, end_mark) + + def scan_plain_spaces(self, indent, start_mark): + # See the specification for details. + # The specification is really confusing about tabs in plain scalars. + # We just forbid them completely. Do not use tabs in YAML! + chunks = [] + length = 0 + while self.peek(length) in ' ': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch in '\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + self.allow_simple_key = True + prefix = self.prefix(3) + if (prefix == '---' or prefix == '...') \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return + breaks = [] + while self.peek() in ' \r\n\x85\u2028\u2029': + if self.peek() == ' ': + self.forward() + else: + breaks.append(self.scan_line_break()) + prefix = self.prefix(3) + if (prefix == '---' or prefix == '...') \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return + if line_break != '\n': + chunks.append(line_break) + elif not breaks: + chunks.append(' ') + chunks.extend(breaks) + elif whitespaces: + chunks.append(whitespaces) + return chunks + + def scan_tag_handle(self, name, start_mark): + # See the specification for details. + # For some strange reasons, the specification does not allow '_' in + # tag handles. I have allowed it anyway. + ch = self.peek() + if ch != '!': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch, self.get_mark()) + length = 1 + ch = self.peek(length) + if ch != ' ': + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_': + length += 1 + ch = self.peek(length) + if ch != '!': + self.forward(length) + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch, self.get_mark()) + length += 1 + value = self.prefix(length) + self.forward(length) + return value + + def scan_tag_uri(self, name, start_mark): + # See the specification for details. + # Note: we do not check if URI is well-formed. + chunks = [] + length = 0 + ch = self.peek(length) + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-;/?:@&=+$,_.!~*\'()[]%': + if ch == '%': + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + chunks.append(self.scan_uri_escapes(name, start_mark)) + else: + length += 1 + ch = self.peek(length) + if length: + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + if not chunks: + raise ScannerError("while parsing a %s" % name, start_mark, + "expected URI, but found %r" % ch, self.get_mark()) + return ''.join(chunks) + + def scan_uri_escapes(self, name, start_mark): + # See the specification for details. + codes = [] + mark = self.get_mark() + while self.peek() == '%': + self.forward() + for k in range(2): + if self.peek(k) not in '0123456789ABCDEFabcdef': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected URI escape sequence of 2 hexdecimal numbers, but found %r" + % self.peek(k), self.get_mark()) + codes.append(int(self.prefix(2), 16)) + self.forward(2) + try: + value = bytes(codes).decode('utf-8') + except UnicodeDecodeError as exc: + raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark) + return value + + def scan_line_break(self): + # Transforms: + # '\r\n' : '\n' + # '\r' : '\n' + # '\n' : '\n' + # '\x85' : '\n' + # '\u2028' : '\u2028' + # '\u2029 : '\u2029' + # default : '' + ch = self.peek() + if ch in '\r\n\x85': + if self.prefix(2) == '\r\n': + self.forward(2) + else: + self.forward() + return '\n' + elif ch in '\u2028\u2029': + self.forward() + return ch + return '' diff --git a/pipenv/patched/yaml3/serializer.py b/pipenv/patched/yaml3/serializer.py new file mode 100644 index 00000000..fe911e67 --- /dev/null +++ b/pipenv/patched/yaml3/serializer.py @@ -0,0 +1,111 @@ + +__all__ = ['Serializer', 'SerializerError'] + +from .error import YAMLError +from .events import * +from .nodes import * + +class SerializerError(YAMLError): + pass + +class Serializer: + + ANCHOR_TEMPLATE = 'id%03d' + + def __init__(self, encoding=None, + explicit_start=None, explicit_end=None, version=None, tags=None): + self.use_encoding = encoding + self.use_explicit_start = explicit_start + self.use_explicit_end = explicit_end + self.use_version = version + self.use_tags = tags + self.serialized_nodes = {} + self.anchors = {} + self.last_anchor_id = 0 + self.closed = None + + def open(self): + if self.closed is None: + self.emit(StreamStartEvent(encoding=self.use_encoding)) + self.closed = False + elif self.closed: + raise SerializerError("serializer is closed") + else: + raise SerializerError("serializer is already opened") + + def close(self): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif not self.closed: + self.emit(StreamEndEvent()) + self.closed = True + + #def __del__(self): + # self.close() + + def serialize(self, node): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif self.closed: + raise SerializerError("serializer is closed") + self.emit(DocumentStartEvent(explicit=self.use_explicit_start, + version=self.use_version, tags=self.use_tags)) + self.anchor_node(node) + self.serialize_node(node, None, None) + self.emit(DocumentEndEvent(explicit=self.use_explicit_end)) + self.serialized_nodes = {} + self.anchors = {} + self.last_anchor_id = 0 + + def anchor_node(self, node): + if node in self.anchors: + if self.anchors[node] is None: + self.anchors[node] = self.generate_anchor(node) + else: + self.anchors[node] = None + if isinstance(node, SequenceNode): + for item in node.value: + self.anchor_node(item) + elif isinstance(node, MappingNode): + for key, value in node.value: + self.anchor_node(key) + self.anchor_node(value) + + def generate_anchor(self, node): + self.last_anchor_id += 1 + return self.ANCHOR_TEMPLATE % self.last_anchor_id + + def serialize_node(self, node, parent, index): + alias = self.anchors[node] + if node in self.serialized_nodes: + self.emit(AliasEvent(alias)) + else: + self.serialized_nodes[node] = True + self.descend_resolver(parent, index) + if isinstance(node, ScalarNode): + detected_tag = self.resolve(ScalarNode, node.value, (True, False)) + default_tag = self.resolve(ScalarNode, node.value, (False, True)) + implicit = (node.tag == detected_tag), (node.tag == default_tag) + self.emit(ScalarEvent(alias, node.tag, implicit, node.value, + style=node.style)) + elif isinstance(node, SequenceNode): + implicit = (node.tag + == self.resolve(SequenceNode, node.value, True)) + self.emit(SequenceStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + index = 0 + for item in node.value: + self.serialize_node(item, node, index) + index += 1 + self.emit(SequenceEndEvent()) + elif isinstance(node, MappingNode): + implicit = (node.tag + == self.resolve(MappingNode, node.value, True)) + self.emit(MappingStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + for key, value in node.value: + self.serialize_node(key, node, None) + self.serialize_node(value, node, key) + self.emit(MappingEndEvent()) + self.ascend_resolver() + diff --git a/pipenv/patched/yaml3/tokens.py b/pipenv/patched/yaml3/tokens.py new file mode 100644 index 00000000..4d0b48a3 --- /dev/null +++ b/pipenv/patched/yaml3/tokens.py @@ -0,0 +1,104 @@ + +class Token(object): + def __init__(self, start_mark, end_mark): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in self.__dict__ + if not key.endswith('_mark')] + attributes.sort() + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +#class BOMToken(Token): +# id = '' + +class DirectiveToken(Token): + id = '' + def __init__(self, name, value, start_mark, end_mark): + self.name = name + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class DocumentStartToken(Token): + id = '' + +class DocumentEndToken(Token): + id = '' + +class StreamStartToken(Token): + id = '' + def __init__(self, start_mark=None, end_mark=None, + encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndToken(Token): + id = '' + +class BlockSequenceStartToken(Token): + id = '' + +class BlockMappingStartToken(Token): + id = '' + +class BlockEndToken(Token): + id = '' + +class FlowSequenceStartToken(Token): + id = '[' + +class FlowMappingStartToken(Token): + id = '{' + +class FlowSequenceEndToken(Token): + id = ']' + +class FlowMappingEndToken(Token): + id = '}' + +class KeyToken(Token): + id = '?' + +class ValueToken(Token): + id = ':' + +class BlockEntryToken(Token): + id = '-' + +class FlowEntryToken(Token): + id = ',' + +class AliasToken(Token): + id = '' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class AnchorToken(Token): + id = '' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class TagToken(Token): + id = '' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class ScalarToken(Token): + id = '' + def __init__(self, value, plain, start_mark, end_mark, style=None): + self.value = value + self.plain = plain + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + diff --git a/pipenv/vendor/certifi/__init__.py b/pipenv/vendor/certifi/__init__.py index 0d59a056..1e2dfac7 100644 --- a/pipenv/vendor/certifi/__init__.py +++ b/pipenv/vendor/certifi/__init__.py @@ -1,3 +1,3 @@ -from .core import where +from .core import contents, where -__version__ = "2019.11.28" +__version__ = "2020.04.05.1" diff --git a/pipenv/vendor/certifi/__main__.py b/pipenv/vendor/certifi/__main__.py index 5f1da0dd..8945b5da 100644 --- a/pipenv/vendor/certifi/__main__.py +++ b/pipenv/vendor/certifi/__main__.py @@ -1,2 +1,12 @@ -from certifi import where -print(where()) +import argparse + +from certifi import contents, where + +parser = argparse.ArgumentParser() +parser.add_argument("-c", "--contents", action="store_true") +args = parser.parse_args() + +if args.contents: + print(contents()) +else: + print(where()) diff --git a/pipenv/vendor/certifi/cacert.pem b/pipenv/vendor/certifi/cacert.pem index a4758ef3..ece147c9 100644 --- a/pipenv/vendor/certifi/cacert.pem +++ b/pipenv/vendor/certifi/cacert.pem @@ -2140,6 +2140,45 @@ t/2jioSgrGK+KwmHNPBqAbubKVY8/gA3zyNs8U6qtnRGEmyR7jTV7JqR50S+kDFy SjnRBUkLp7Y3gaVdjKozXoEofKd9J+sAro03 -----END CERTIFICATE----- +# Issuer: CN=EC-ACC O=Agencia Catalana de Certificacio (NIF Q-0801176-I) OU=Serveis Publics de Certificacio/Vegeu https://www.catcert.net/verarrel (c)03/Jerarquia Entitats de Certificacio Catalanes +# Subject: CN=EC-ACC O=Agencia Catalana de Certificacio (NIF Q-0801176-I) OU=Serveis Publics de Certificacio/Vegeu https://www.catcert.net/verarrel (c)03/Jerarquia Entitats de Certificacio Catalanes +# Label: "EC-ACC" +# Serial: -23701579247955709139626555126524820479 +# MD5 Fingerprint: eb:f5:9d:29:0d:61:f9:42:1f:7c:c2:ba:6d:e3:15:09 +# SHA1 Fingerprint: 28:90:3a:63:5b:52:80:fa:e6:77:4c:0b:6d:a7:d6:ba:a6:4a:f2:e8 +# SHA256 Fingerprint: 88:49:7f:01:60:2f:31:54:24:6a:e2:8c:4d:5a:ef:10:f1:d8:7e:bb:76:62:6f:4a:e0:b7:f9:5b:a7:96:87:99 +-----BEGIN CERTIFICATE----- +MIIFVjCCBD6gAwIBAgIQ7is969Qh3hSoYqwE893EATANBgkqhkiG9w0BAQUFADCB +8zELMAkGA1UEBhMCRVMxOzA5BgNVBAoTMkFnZW5jaWEgQ2F0YWxhbmEgZGUgQ2Vy +dGlmaWNhY2lvIChOSUYgUS0wODAxMTc2LUkpMSgwJgYDVQQLEx9TZXJ2ZWlzIFB1 +YmxpY3MgZGUgQ2VydGlmaWNhY2lvMTUwMwYDVQQLEyxWZWdldSBodHRwczovL3d3 +dy5jYXRjZXJ0Lm5ldC92ZXJhcnJlbCAoYykwMzE1MDMGA1UECxMsSmVyYXJxdWlh +IEVudGl0YXRzIGRlIENlcnRpZmljYWNpbyBDYXRhbGFuZXMxDzANBgNVBAMTBkVD +LUFDQzAeFw0wMzAxMDcyMzAwMDBaFw0zMTAxMDcyMjU5NTlaMIHzMQswCQYDVQQG +EwJFUzE7MDkGA1UEChMyQWdlbmNpYSBDYXRhbGFuYSBkZSBDZXJ0aWZpY2FjaW8g +KE5JRiBRLTA4MDExNzYtSSkxKDAmBgNVBAsTH1NlcnZlaXMgUHVibGljcyBkZSBD +ZXJ0aWZpY2FjaW8xNTAzBgNVBAsTLFZlZ2V1IGh0dHBzOi8vd3d3LmNhdGNlcnQu +bmV0L3ZlcmFycmVsIChjKTAzMTUwMwYDVQQLEyxKZXJhcnF1aWEgRW50aXRhdHMg +ZGUgQ2VydGlmaWNhY2lvIENhdGFsYW5lczEPMA0GA1UEAxMGRUMtQUNDMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsyLHT+KXQpWIR4NA9h0X84NzJB5R +85iKw5K4/0CQBXCHYMkAqbWUZRkiFRfCQ2xmRJoNBD45b6VLeqpjt4pEndljkYRm +4CgPukLjbo73FCeTae6RDqNfDrHrZqJyTxIThmV6PttPB/SnCWDaOkKZx7J/sxaV +HMf5NLWUhdWZXqBIoH7nF2W4onW4HvPlQn2v7fOKSGRdghST2MDk/7NQcvJ29rNd +QlB50JQ+awwAvthrDk4q7D7SzIKiGGUzE3eeml0aE9jD2z3Il3rucO2n5nzbcc8t +lGLfbdb1OL4/pYUKGbio2Al1QnDE6u/LDsg0qBIimAy4E5S2S+zw0JDnJwIDAQAB +o4HjMIHgMB0GA1UdEQQWMBSBEmVjX2FjY0BjYXRjZXJ0Lm5ldDAPBgNVHRMBAf8E +BTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUoMOLRKo3pUW/l4Ba0fF4 +opvpXY0wfwYDVR0gBHgwdjB0BgsrBgEEAfV4AQMBCjBlMCwGCCsGAQUFBwIBFiBo +dHRwczovL3d3dy5jYXRjZXJ0Lm5ldC92ZXJhcnJlbDA1BggrBgEFBQcCAjApGidW +ZWdldSBodHRwczovL3d3dy5jYXRjZXJ0Lm5ldC92ZXJhcnJlbCAwDQYJKoZIhvcN +AQEFBQADggEBAKBIW4IB9k1IuDlVNZyAelOZ1Vr/sXE7zDkJlF7W2u++AVtd0x7Y +/X1PzaBB4DSTv8vihpw3kpBWHNzrKQXlxJ7HNd+KDM3FIUPpqojlNcAZQmNaAl6k +SBg6hW/cnbw/nZzBh7h6YQjpdwt/cKt63dmXLGQehb+8dJahw3oS7AwaboMMPOhy +Rp/7SNVel+axofjk70YllJyJ22k4vuxcDlbHZVHlUIiIv0LVKz3l+bqeLrPK9HOS +Agu+TGbrIP65y7WZf+a2E/rKS03Z7lNGBjvGTq2TWoF+bCpLagVFjPIhpDGQh2xl +nJ2lYJU6Un/10asIbvPuW/mIPX64b24D5EI= +-----END CERTIFICATE----- + # Issuer: CN=Hellenic Academic and Research Institutions RootCA 2011 O=Hellenic Academic and Research Institutions Cert. Authority # Subject: CN=Hellenic Academic and Research Institutions RootCA 2011 O=Hellenic Academic and Research Institutions Cert. Authority # Label: "Hellenic Academic and Research Institutions RootCA 2011" diff --git a/pipenv/vendor/certifi/core.py b/pipenv/vendor/certifi/core.py index 7271acf4..56b52a3c 100644 --- a/pipenv/vendor/certifi/core.py +++ b/pipenv/vendor/certifi/core.py @@ -4,12 +4,27 @@ certifi.py ~~~~~~~~~~ -This module returns the installation location of cacert.pem. +This module returns the installation location of cacert.pem or its contents. """ import os +try: + from importlib.resources import read_text +except ImportError: + # This fallback will work for Python versions prior to 3.7 that lack the + # importlib.resources module but relies on the existing `where` function + # so won't address issues with environments like PyOxidizer that don't set + # __file__ on modules. + def read_text(_module, _path, encoding="ascii"): + with open(where(), "r", encoding=encoding) as data: + return data.read() + def where(): f = os.path.dirname(__file__) - return os.path.join(f, 'cacert.pem') + return os.path.join(f, "cacert.pem") + + +def contents(): + return read_text("certifi", "cacert.pem", encoding="ascii") diff --git a/pipenv/vendor/dparse/LICENSE b/pipenv/vendor/dparse/LICENSE new file mode 100644 index 00000000..49d08568 --- /dev/null +++ b/pipenv/vendor/dparse/LICENSE @@ -0,0 +1,11 @@ + +MIT License + +Copyright (c) 2017, Jannis Gebauer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/pipenv/vendor/dparse/__init__.py b/pipenv/vendor/dparse/__init__.py new file mode 100644 index 00000000..295f38ee --- /dev/null +++ b/pipenv/vendor/dparse/__init__.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import +"""Top-level package for Dependency Parser.""" + +__author__ = """Jannis Gebauer""" +__email__ = 'jay@pyup.io' +__version__ = '0.5.0' + +from .parser import parse diff --git a/pipenv/vendor/dparse/dependencies.py b/pipenv/vendor/dparse/dependencies.py new file mode 100644 index 00000000..8881a9bc --- /dev/null +++ b/pipenv/vendor/dparse/dependencies.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import + +import json + +from . import filetypes, errors + + +class Dependency(object): + """ + + """ + + def __init__(self, name, specs, line, source="pypi", meta={}, extras=[], line_numbers=None, index_server=None, hashes=(), dependency_type=None, section=None): + """ + + :param name: + :param specs: + :param line: + :param source: + :param extras: + :param line_numbers: + :param index_server: + :param hashes: + :param dependency_type: + """ + self.name = name + self.key = name.lower().replace("_", "-") + self.specs = specs + self.line = line + self.source = source + self.meta = meta + self.line_numbers = line_numbers + self.index_server = index_server + self.hashes = hashes + self.dependency_type = dependency_type + self.extras = extras + self.section = section + + def __str__(self): # pragma: no cover + """ + + :return: + """ + return "Dependency({name}, {specs}, {line})".format( + name=self.name, + specs=self.specs, + line=self.line + ) + + def serialize(self): + """ + + :return: + """ + return { + "name": self.name, + "specs": self.specs, + "line": self.line, + "source": self.source, + "meta": self.meta, + "line_numbers": self.line_numbers, + "index_server": self.index_server, + "hashes": self.hashes, + "dependency_type": self.dependency_type, + "extras": self.extras, + "section": self.section + } + + @classmethod + def deserialize(cls, d): + """ + + :param d: + :return: + """ + return cls(**d) + + @property + def full_name(self): + """ + + :return: + """ + if self.extras: + return "{}[{}]".format(self.name, ",".join(self.extras)) + return self.name + + +class DependencyFile(object): + """ + + """ + + def __init__(self, content, path=None, sha=None, file_type=None, marker=((), ()), parser=None): + """ + + :param content: + :param path: + :param sha: + :param marker: + :param file_type: + :param parser: + """ + self.content = content + self.file_type = file_type + self.path = path + self.sha = sha + self.marker = marker + + self.dependencies = [] + self.resolved_files = [] + self.is_valid = False + self.file_marker, self.line_marker = marker + + if parser: + self.parser = parser + else: + from . import parser as parser_class + if file_type is not None: + if file_type == filetypes.requirements_txt: + self.parser = parser_class.RequirementsTXTParser + elif file_type == filetypes.tox_ini: + self.parser = parser_class.ToxINIParser + elif file_type == filetypes.conda_yml: + self.parser = parser_class.CondaYMLParser + elif file_type == filetypes.pipfile: + self.parser = parser_class.PipfileParser + elif file_type == filetypes.pipfile_lock: + self.parser = parser_class.PipfileLockParser + elif file_type == filetypes.setup_cfg: + self.parser = parser_class.SetupCfgParser + + elif path is not None: + if path.endswith(".txt"): + self.parser = parser_class.RequirementsTXTParser + elif path.endswith(".yml"): + self.parser = parser_class.CondaYMLParser + elif path.endswith(".ini"): + self.parser = parser_class.ToxINIParser + elif path.endswith("Pipfile"): + self.parser = parser_class.PipfileParser + elif path.endswith("Pipfile.lock"): + self.parser = parser_class.PipfileLockParser + elif path.endswith("setup.cfg"): + self.parser = parser_class.SetupCfgParser + + if not hasattr(self, "parser"): + raise errors.UnknownDependencyFileError + + self.parser = self.parser(self) + + def serialize(self): + """ + + :return: + """ + return { + "file_type": self.file_type, + "content": self.content, + "path": self.path, + "sha": self.sha, + "dependencies": [dep.serialize() for dep in self.dependencies] + } + + @classmethod + def deserialize(cls, d): + """ + + :param d: + :return: + """ + dependencies = [Dependency.deserialize(dep) for dep in d.pop("dependencies", [])] + instance = cls(**d) + instance.dependencies = dependencies + return instance + + def json(self): # pragma: no cover + """ + + :return: + """ + return json.dumps(self.serialize(), indent=2) + + def parse(self): + """ + + :return: + """ + if self.parser.is_marked_file: + self.is_valid = False + return self + self.parser.parse() + + self.is_valid = len(self.dependencies) > 0 or len(self.resolved_files) > 0 + return self diff --git a/pipenv/vendor/dparse/errors.py b/pipenv/vendor/dparse/errors.py new file mode 100644 index 00000000..b7a65430 --- /dev/null +++ b/pipenv/vendor/dparse/errors.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import + +class UnknownDependencyFileError(Exception): + """ + + """ + pass diff --git a/pipenv/vendor/dparse/filetypes.py b/pipenv/vendor/dparse/filetypes.py new file mode 100644 index 00000000..df8a913e --- /dev/null +++ b/pipenv/vendor/dparse/filetypes.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import + +requirements_txt = "requirements.txt" +conda_yml = "conda.yml" +setup_cfg = "setup.cfg" +tox_ini = "tox.ini" +pipfile = "Pipfile" +pipfile_lock = "Pipfile.lock" diff --git a/pipenv/vendor/dparse/parser.py b/pipenv/vendor/dparse/parser.py new file mode 100644 index 00000000..9b2a0728 --- /dev/null +++ b/pipenv/vendor/dparse/parser.py @@ -0,0 +1,427 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import +from collections import OrderedDict +import re +import yaml + +from io import StringIO + +from six.moves.configparser import SafeConfigParser, NoOptionError + + +from .regex import URL_REGEX, HASH_REGEX + +from .dependencies import DependencyFile, Dependency +from packaging.requirements import Requirement as PackagingRequirement, InvalidRequirement +from . import filetypes +import toml +from packaging.specifiers import SpecifierSet +import json + + +# this is a backport from setuptools 26.1 +def setuptools_parse_requirements_backport(strs): # pragma: no cover + # Copyright (C) 2016 Jason R Coombs + # + # Permission is hereby granted, free of charge, to any person obtaining a copy of + # this software and associated documentation files (the "Software"), to deal in + # the Software without restriction, including without limitation the rights to + # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + # of the Software, and to permit persons to whom the Software is furnished to do + # so, subject to the following conditions: + # + # The above copyright notice and this permission notice shall be included in all + # copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + # SOFTWARE. + """Yield ``Requirement`` objects for each specification in `strs` + + `strs` must be a string, or a (possibly-nested) iterable thereof. + """ + # create a steppable iterator, so we can handle \-continuations + def yield_lines(strs): + """Yield non-empty/non-comment lines of a string or sequence""" + if isinstance(strs, str): + for s in strs.splitlines(): + s = s.strip() + # skip blank lines/comments + if s and not s.startswith('#'): + yield s + else: + for ss in strs: + for s in yield_lines(ss): + yield s + lines = iter(yield_lines(strs)) + + for line in lines: + # Drop comments -- a hash without a space may be in a URL. + if ' #' in line: + line = line[:line.find(' #')] + # If there is a line continuation, drop it, and append the next line. + if line.endswith('\\'): + line = line[:-2].strip() + line += next(lines) + yield PackagingRequirement(line) + + +class RequirementsTXTLineParser(object): + """ + + """ + + @classmethod + def parse(cls, line): + """ + + :param line: + :return: + """ + try: + # setuptools requires a space before the comment. If this isn't the case, add it. + if "\t#" in line: + parsed, = setuptools_parse_requirements_backport(line.replace("\t#", "\t #")) + else: + parsed, = setuptools_parse_requirements_backport(line) + except InvalidRequirement: + return None + dep = Dependency( + name=parsed.name, + specs=parsed.specifier, + line=line, + extras=parsed.extras, + dependency_type=filetypes.requirements_txt + ) + return dep + + +class Parser(object): + """ + + """ + + def __init__(self, obj): + """ + + :param obj: + """ + self.obj = obj + self._lines = None + + def iter_lines(self, lineno=0): + """ + + :param lineno: + :return: + """ + for line in self.lines[lineno:]: + yield line + + @property + def lines(self): + """ + + :return: + """ + if self._lines is None: + self._lines = self.obj.content.splitlines() + return self._lines + + @property + def is_marked_file(self): + """ + + :return: + """ + for n, line in enumerate(self.iter_lines()): + for marker in self.obj.file_marker: + if marker in line: + return True + if n >= 2: + break + return False + + def is_marked_line(self, line): + """ + + :param line: + :return: + """ + for marker in self.obj.line_marker: + if marker in line: + return True + return False + + @classmethod + def parse_hashes(cls, line): + """ + + :param line: + :return: + """ + hashes = [] + for match in re.finditer(HASH_REGEX, line): + hashes.append(line[match.start():match.end()]) + return re.sub(HASH_REGEX, "", line).strip(), hashes + + @classmethod + def parse_index_server(cls, line): + """ + + :param line: + :return: + """ + matches = URL_REGEX.findall(line) + if matches: + url = matches[0] + return url if url.endswith("/") else url + "/" + return None + + @classmethod + def resolve_file(cls, file_path, line): + """ + + :param file_path: + :param line: + :return: + """ + line = line.replace("-r ", "").replace("--requirement ", "") + parts = file_path.split("/") + if " #" in line: + line = line.split("#")[0].strip() + if len(parts) == 1: + return line + return "/".join(parts[:-1]) + "/" + line + + +class RequirementsTXTParser(Parser): + """ + + """ + + def parse(self): + """ + Parses a requirements.txt-like file + """ + index_server = None + for num, line in enumerate(self.iter_lines()): + line = line.rstrip() + if not line: + continue + if line.startswith('#'): + # comments are lines that start with # only + continue + if line.startswith('-i') or \ + line.startswith('--index-url') or \ + line.startswith('--extra-index-url'): + # this file is using a private index server, try to parse it + index_server = self.parse_index_server(line) + continue + elif self.obj.path and (line.startswith('-r') or line.startswith('--requirement')): + self.obj.resolved_files.append(self.resolve_file(self.obj.path, line)) + elif line.startswith('-f') or line.startswith('--find-links') or \ + line.startswith('--no-index') or line.startswith('--allow-external') or \ + line.startswith('--allow-unverified') or line.startswith('-Z') or \ + line.startswith('--always-unzip'): + continue + elif self.is_marked_line(line): + continue + else: + try: + + parseable_line = line + + # multiline requirements are not parseable + if "\\" in line: + parseable_line = line.replace("\\", "") + for next_line in self.iter_lines(num + 1): + parseable_line += next_line.strip().replace("\\", "") + line += "\n" + next_line + if "\\" in next_line: + continue + break + # ignore multiline requirements if they are marked + if self.is_marked_line(parseable_line): + continue + + hashes = [] + if "--hash" in parseable_line: + parseable_line, hashes = Parser.parse_hashes(parseable_line) + + req = RequirementsTXTLineParser.parse(parseable_line) + if req: + req.hashes = hashes + req.index_server = index_server + # replace the requirements line with the 'real' line + req.line = line + self.obj.dependencies.append(req) + except ValueError: + continue + + +class ToxINIParser(Parser): + """ + + """ + + def parse(self): + """ + + :return: + """ + parser = SafeConfigParser() + parser.readfp(StringIO(self.obj.content)) + for section in parser.sections(): + try: + content = parser.get(section=section, option="deps") + for n, line in enumerate(content.splitlines()): + if self.is_marked_line(line): + continue + if line: + req = RequirementsTXTLineParser.parse(line) + if req: + req.dependency_type = self.obj.file_type + self.obj.dependencies.append(req) + except NoOptionError: + pass + + +class CondaYMLParser(Parser): + """ + + """ + + def parse(self): + """ + + :return: + """ + try: + data = yaml.safe_load(self.obj.content) + if data and 'dependencies' in data and isinstance(data['dependencies'], list): + for dep in data['dependencies']: + if isinstance(dep, dict) and 'pip' in dep: + for n, line in enumerate(dep['pip']): + if self.is_marked_line(line): + continue + req = RequirementsTXTLineParser.parse(line) + if req: + req.dependency_type = self.obj.file_type + self.obj.dependencies.append(req) + except yaml.YAMLError: + pass + + +class PipfileParser(Parser): + + def parse(self): + """ + Parse a Pipfile (as seen in pipenv) + :return: + """ + try: + data = toml.loads(self.obj.content, _dict=OrderedDict) + if data: + for package_type in ['packages', 'dev-packages']: + if package_type in data: + for name, specs in data[package_type].items(): + # skip on VCS dependencies + if not isinstance(specs, str): + continue + if specs == '*': + specs = '' + self.obj.dependencies.append( + Dependency( + name=name, specs=SpecifierSet(specs), + dependency_type=filetypes.pipfile, + line=''.join([name, specs]), + section=package_type + ) + ) + except (toml.TomlDecodeError, IndexError) as e: + pass + +class PipfileLockParser(Parser): + + def parse(self): + """ + Parse a Pipfile.lock (as seen in pipenv) + :return: + """ + try: + data = json.loads(self.obj.content, object_pairs_hook=OrderedDict) + if data: + for package_type in ['default', 'develop']: + if package_type in data: + for name, meta in data[package_type].items(): + # skip VCS dependencies + if 'version' not in meta: + continue + specs = meta['version'] + hashes = meta['hashes'] + self.obj.dependencies.append( + Dependency( + name=name, specs=SpecifierSet(specs), + dependency_type=filetypes.pipfile_lock, + hashes=hashes, + line=''.join([name, specs]), + section=package_type + ) + ) + except ValueError: + pass + + +class SetupCfgParser(Parser): + def parse(self): + parser = SafeConfigParser() + parser.readfp(StringIO(self.obj.content)) + for section in parser.values(): + if section.name == 'options': + options = 'install_requires', 'setup_requires', 'test_require' + for name in options: + content = section.get(name) + if not content: + continue + self._parse_content(content) + elif section.name == 'options.extras_require': + for content in section.values(): + self._parse_content(content) + + def _parse_content(self, content): + for n, line in enumerate(content.splitlines()): + if self.is_marked_line(line): + continue + if line: + req = RequirementsTXTLineParser.parse(line) + if req: + req.dependency_type = self.obj.file_type + self.obj.dependencies.append(req) + + +def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=None): + """ + + :param content: + :param file_type: + :param path: + :param sha: + :param marker: + :param parser: + :return: + """ + dep_file = DependencyFile( + content=content, + path=path, + sha=sha, + marker=marker, + file_type=file_type, + parser=parser + ) + + return dep_file.parse() diff --git a/pipenv/vendor/dparse/regex.py b/pipenv/vendor/dparse/regex.py new file mode 100644 index 00000000..40cc4091 --- /dev/null +++ b/pipenv/vendor/dparse/regex.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import re +# see https://gist.github.com/dperini/729294 +URL_REGEX = re.compile( + # protocol identifier + "(?:(?:https?|ftp)://)" + # user:pass authentication + "(?:\S+(?::\S*)?@)?" + "(?:" + # IP address exclusion + # private & local networks + "(?!(?:10|127)(?:\.\d{1,3}){3})" + "(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})" + "(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})" + # IP address dotted notation octets + # excludes loopback network 0.0.0.0 + # excludes reserved space >= 224.0.0.0 + # excludes network & broadcast addresses + # (first & last IP address of each class) + "(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])" + "(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}" + "(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))" + "|" + # host name + "(?:(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)" + # domain name + "(?:\.(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)*" + # TLD identifier + "(?:\.(?:[a-z\u00a1-\uffff]{2,}))" + ")" + # port number + "(?::\d{2,5})?" + # resource path + "(?:/\S*)?", + re.UNICODE) + +HASH_REGEX = r"--hash[=| ][\w]+:[\w]+" diff --git a/pipenv/vendor/dparse/updater.py b/pipenv/vendor/dparse/updater.py new file mode 100644 index 00000000..77b5ae66 --- /dev/null +++ b/pipenv/vendor/dparse/updater.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals +import re +import json +import tempfile +import toml +import os + + +class RequirementsTXTUpdater(object): + + SUB_REGEX = r"^{}(?=\s*\r?\n?$)" + + @classmethod + def update(cls, content, dependency, version, spec="==", hashes=()): + """ + Updates the requirement to the latest version for the given content and adds hashes + if neccessary. + :param content: str, content + :return: str, updated content + """ + new_line = "{name}{spec}{version}".format(name=dependency.full_name, spec=spec, version=version) + appendix = '' + # leave environment markers intact + if ";" in dependency.line: + # condense multiline, split out the env marker, strip comments and --hashes + new_line += ";" + dependency.line.splitlines()[0].split(";", 1)[1] \ + .split("#")[0].split("--hash")[0].rstrip() + # add the comment + if "#" in dependency.line: + # split the line into parts: requirement and comment + parts = dependency.line.split("#") + requirement, comment = parts[0], "#".join(parts[1:]) + # find all whitespaces between the requirement and the comment + whitespaces = (hex(ord('\t')), hex(ord(' '))) + trailing_whitespace = '' + for c in requirement[::-1]: + if hex(ord(c)) in whitespaces: + trailing_whitespace += c + else: + break + appendix += trailing_whitespace + "#" + comment + # if this is a hashed requirement, add a multiline break before the comment + if dependency.hashes and not new_line.endswith("\\"): + new_line += " \\" + # if this is a hashed requirement, add the hashes + if hashes: + for n, new_hash in enumerate(hashes): + new_line += "\n --hash={method}:{hash}".format( + method=new_hash['method'], + hash=new_hash['hash'] + ) + # append a new multiline break if this is not the last line + if len(hashes) > n + 1: + new_line += " \\" + new_line += appendix + + regex = cls.SUB_REGEX.format(re.escape(dependency.line)) + + return re.sub(regex, new_line, content, flags=re.MULTILINE) + + +class CondaYMLUpdater(RequirementsTXTUpdater): + + SUB_REGEX = r"{}(?=\s*\r?\n?$)" + + +class ToxINIUpdater(CondaYMLUpdater): + pass + + +class SetupCFGUpdater(CondaYMLUpdater): + pass + + +class PipfileUpdater(object): + @classmethod + def update(cls, content, dependency, version, spec="==", hashes=()): + data = toml.loads(content) + if data: + for package_type in ['packages', 'dev-packages']: + if package_type in data: + if dependency.full_name in data[package_type]: + data[package_type][dependency.full_name] = "{spec}{version}".format( + spec=spec, version=version + ) + try: + from pipenv.project import Project + except ImportError: + raise ImportError("Updating a Pipfile requires the pipenv extra to be installed. Install it with " + "pip install dparse[pipenv]") + pipfile = tempfile.NamedTemporaryFile(delete=False) + p = Project(chdir=False) + p.write_toml(data=data, path=pipfile.name) + data = open(pipfile.name).read() + os.remove(pipfile.name) + return data + + +class PipfileLockUpdater(object): + @classmethod + def update(cls, content, dependency, version, spec="==", hashes=()): + data = json.loads(content) + if data: + for package_type in ['default', 'develop']: + if package_type in data: + if dependency.full_name in data[package_type]: + data[package_type][dependency.full_name] = { + 'hashes': [ + "{method}:{hash}".format( + hash=h['hash'], + method=h['method'] + ) for h in hashes + ], + 'version': "{spec}{version}".format( + spec=spec, version=version + ) + } + return json.dumps(data, indent=4, separators=(',', ': ')) + "\n" diff --git a/pipenv/vendor/importlib_metadata/__init__.py b/pipenv/vendor/importlib_metadata/__init__.py index 089fca97..95a08ba0 100644 --- a/pipenv/vendor/importlib_metadata/__init__.py +++ b/pipenv/vendor/importlib_metadata/__init__.py @@ -28,6 +28,7 @@ from ._compat import ( MetaPathFinder, email_message_from_string, PyPy_repr, + unique_ordered, ) from importlib import import_module from itertools import starmap @@ -95,6 +96,16 @@ class EntryPoint( attrs = filter(None, (match.group('attr') or '').split('.')) return functools.reduce(getattr, attrs, module) + @property + def module(self): + match = self.pattern.match(self.value) + return match.group('module') + + @property + def attr(self): + match = self.pattern.match(self.value) + return match.group('attr') + @property def extras(self): match = self.pattern.match(self.value) @@ -425,8 +436,8 @@ class FastPath: names = zip_path.root.namelist() self.joinpath = zip_path.joinpath - return ( - posixpath.split(child)[0] + return unique_ordered( + child.split(posixpath.sep, 1)[0] for child in names ) diff --git a/pipenv/vendor/importlib_metadata/_compat.py b/pipenv/vendor/importlib_metadata/_compat.py index 99b4005e..f59b57db 100644 --- a/pipenv/vendor/importlib_metadata/_compat.py +++ b/pipenv/vendor/importlib_metadata/_compat.py @@ -15,9 +15,11 @@ if sys.version_info > (3,): # pragma: nocover NotADirectoryError = builtins.NotADirectoryError PermissionError = builtins.PermissionError map = builtins.map + from itertools import filterfalse else: # pragma: nocover from backports.configparser import ConfigParser from itertools import imap as map # type: ignore + from itertools import ifilterfalse as filterfalse import contextlib2 as contextlib FileNotFoundError = IOError, OSError IsADirectoryError = IOError, OSError @@ -131,3 +133,18 @@ class PyPy_repr: if affected: # pragma: nocover __repr__ = __compat_repr__ del affected + + +# from itertools recipes +def unique_everseen(iterable): # pragma: nocover + "List unique elements, preserving order. Remember all elements ever seen." + seen = set() + seen_add = seen.add + + for element in filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + + +unique_ordered = ( + unique_everseen if sys.version_info < (3, 7) else dict.fromkeys) diff --git a/pipenv/vendor/importlib_metadata/docs/changelog.rst b/pipenv/vendor/importlib_metadata/docs/changelog.rst index 9dbaf96a..d638e38d 100644 --- a/pipenv/vendor/importlib_metadata/docs/changelog.rst +++ b/pipenv/vendor/importlib_metadata/docs/changelog.rst @@ -2,6 +2,17 @@ importlib_metadata NEWS ========================= +v1.6.0 +====== + +* Added ``module`` and ``attr`` attributes to ``EntryPoint`` + +v1.5.2 +====== + +* Fix redundant entries from ``FastPath.zip_children``. + Closes #117. + v1.5.1 ====== diff --git a/pipenv/vendor/importlib_metadata/docs/using.rst b/pipenv/vendor/importlib_metadata/docs/using.rst index 6da2bb1c..d1ca7658 100644 --- a/pipenv/vendor/importlib_metadata/docs/using.rst +++ b/pipenv/vendor/importlib_metadata/docs/using.rst @@ -70,7 +70,9 @@ Entry points The ``entry_points()`` function returns a dictionary of all entry points, keyed by group. Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value:: +a ``.load()`` method to resolve the value. There are also ``.module``, +``.attr``, and ``.extras`` attributes for getting the components of the +``.value`` attribute:: >>> eps = entry_points() >>> list(eps) @@ -79,6 +81,12 @@ a ``.load()`` method to resolve the value:: >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] >>> wheel EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') + >>> wheel.module + 'wheel.cli' + >>> wheel.attr + 'main' + >>> wheel.extras + [] >>> main = wheel.load() >>> main diff --git a/pipenv/vendor/importlib_metadata/tests/test_main.py b/pipenv/vendor/importlib_metadata/tests/test_main.py index 131edcea..876203f6 100644 --- a/pipenv/vendor/importlib_metadata/tests/test_main.py +++ b/pipenv/vendor/importlib_metadata/tests/test_main.py @@ -250,3 +250,9 @@ class TestEntryPoints(unittest.TestCase): """ with self.assertRaises(Exception): json.dumps(self.ep) + + def test_module(self): + assert self.ep.module == 'value' + + def test_attr(self): + assert self.ep.attr is None diff --git a/pipenv/vendor/importlib_metadata/tests/test_zip.py b/pipenv/vendor/importlib_metadata/tests/test_zip.py index 8cbba63a..515f593d 100644 --- a/pipenv/vendor/importlib_metadata/tests/test_zip.py +++ b/pipenv/vendor/importlib_metadata/tests/test_zip.py @@ -1,7 +1,10 @@ import sys import unittest -from .. import distribution, entry_points, files, PackageNotFoundError, version +from .. import ( + distribution, entry_points, files, PackageNotFoundError, + version, distributions, + ) try: from importlib.resources import path @@ -52,6 +55,10 @@ class TestZip(unittest.TestCase): path = str(file.dist.locate_file(file)) assert '.whl/' in path, path + def test_one_distribution(self): + dists = list(distributions(path=sys.path[:1])) + assert len(dists) == 1 + class TestEgg(TestZip): def setUp(self): diff --git a/pipenv/vendor/pep517/__init__.py b/pipenv/vendor/pep517/__init__.py index d3705f64..7355b68a 100644 --- a/pipenv/vendor/pep517/__init__.py +++ b/pipenv/vendor/pep517/__init__.py @@ -1,4 +1,4 @@ """Wrappers to build Python packages using PEP 517 hooks """ -__version__ = '0.8.1' +__version__ = '0.8.2' diff --git a/pipenv/vendor/pep517/_in_process.py b/pipenv/vendor/pep517/_in_process.py index 1589a6ca..a536b03e 100644 --- a/pipenv/vendor/pep517/_in_process.py +++ b/pipenv/vendor/pep517/_in_process.py @@ -14,6 +14,7 @@ Results: """ from glob import glob from importlib import import_module +import json import os import os.path from os.path import join as pjoin @@ -22,8 +23,30 @@ import shutil import sys import traceback -# This is run as a script, not a module, so it can't do a relative import -import compat +# This file is run as a script, and `import compat` is not zip-safe, so we +# include write_json() and read_json() from compat.py. +# +# Handle reading and writing JSON in UTF-8, on Python 3 and 2. + +if sys.version_info[0] >= 3: + # Python 3 + def write_json(obj, path, **kwargs): + with open(path, 'w', encoding='utf-8') as f: + json.dump(obj, f, **kwargs) + + def read_json(path): + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + +else: + # Python 2 + def write_json(obj, path, **kwargs): + with open(path, 'wb') as f: + json.dump(obj, f, encoding='utf-8', **kwargs) + + def read_json(path): + with open(path, 'rb') as f: + return json.load(f) class BackendUnavailable(Exception): @@ -233,7 +256,7 @@ def main(): sys.exit("Unknown hook: %s" % hook_name) hook = globals()[hook_name] - hook_input = compat.read_json(pjoin(control_dir, 'input.json')) + hook_input = read_json(pjoin(control_dir, 'input.json')) json_out = {'unsupported': False, 'return_val': None} try: @@ -250,7 +273,7 @@ def main(): except HookMissing: json_out['hook_missing'] = True - compat.write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) + write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) if __name__ == '__main__': diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py index 7feca147..3a0188c9 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py @@ -5,7 +5,7 @@ import sys from . import shims -__version__ = "0.5.1" +__version__ = "0.5.2" if "pip_shims" in sys.modules: diff --git a/pipenv/vendor/pip_shims/compat.py b/pipenv/vendor/pip_shims/compat.py index ed99d970..0c125321 100644 --- a/pipenv/vendor/pip_shims/compat.py +++ b/pipenv/vendor/pip_shims/compat.py @@ -19,6 +19,8 @@ from packaging import specifiers from .environment import MYPY_RUNNING from .utils import ( call_function_with_correct_args, + filter_allowed_args, + get_allowed_args, get_method_args, nullcontext, suppress_setattr, @@ -65,6 +67,7 @@ if MYPY_RUNNING: TCommandInstance = TypeVar("TCommandInstance") TCmdDict = Dict[str, Union[Tuple[str, str, str], TCommandInstance]] TInstallRequirement = TypeVar("TInstallRequirement") + TFormatControl = TypeVar("TFormatControl") TShimmedCmdDict = Union[TShim, TCmdDict] TWheelCache = TypeVar("TWheelCache") TPreparer = TypeVar("TPreparer") @@ -352,11 +355,11 @@ def ensure_resolution_dirs(**kwargs): @contextlib.contextmanager def wheel_cache( - wheel_cache_provider, # type: TShimmedFunc - tempdir_manager_provider, # type: TShimmedFunc - cache_dir, # type: str + cache_dir=None, # type: str format_control=None, # type: Any + wheel_cache_provider=None, # type: TShimmedFunc format_control_provider=None, # type: Optional[TShimmedFunc] + tempdir_manager_provider=None, # type: TShimmedFunc ): tempdir_manager_provider = resolve_possible_shim(tempdir_manager_provider) wheel_cache_provider = resolve_possible_shim(wheel_cache_provider) @@ -490,6 +493,7 @@ def get_requirement_set( cache_dir=None, # type: Optional[str] options=None, # type: Optional[Values] install_cmd_provider=None, # type: Optional[TShimmedFunc] + wheel_cache_provider=None, # type: Optional[TShimmedFunc] ): # (...) -> TRequirementSet """ @@ -499,6 +503,8 @@ def get_requirement_set( invalid parameters will be ignored if they are not needed to generate a requirement set on the current pip version. + :param :class:`~pip_shims.models.ShimmedPathCollection` wheel_cache_provider: A + context manager provider which resolves to a `WheelCache` instance :param install_command: A :class:`~pip._internal.commands.install.InstallCommand` instance which is used to generate the finder. :param :class:`~pip_shims.models.ShimmedPathCollection` req_set_provider: A provider @@ -538,6 +544,7 @@ def get_requirement_set( :return: A new requirement set instance :rtype: :class:`~pip._internal.req.req_set.RequirementSet` """ + wheel_cache_provider = resolve_possible_shim(wheel_cache_provider) req_set_provider = resolve_possible_shim(req_set_provider) if install_command is None: install_cmd_provider = resolve_possible_shim(install_cmd_provider) @@ -565,10 +572,13 @@ def get_requirement_set( ) if session is None and "session" in required_args: session = get_session(install_cmd=install_command, options=options) - results["wheel_cache"] = wheel_cache - results["session"] = session - results["wheel_download_dir"] = wheel_download_dir - return call_function_with_correct_args(req_set_provider, **results) + with ExitStack() as stack: + if wheel_cache is None: + wheel_cache = stack.enter_context(wheel_cache_provider(cache_dir=cache_dir)) + results["wheel_cache"] = wheel_cache + results["session"] = session + results["wheel_download_dir"] = wheel_download_dir + return call_function_with_correct_args(req_set_provider, **results) def get_package_finder( @@ -665,11 +675,11 @@ def get_package_finder( ): if target_python and not received_python: tags = target_python.get_tags() - version_impl = set([t[0] for t in tags]) + version_impl = {t[0] for t in tags} # impls = set([v[:2] for v in version_impl]) # impls.remove("py") # impl = next(iter(impls), "py") if not target_python - versions = set([v[2:] for v in version_impl]) + versions = {v[2:] for v in version_impl} build_kwargs.update( { "platform": target_python.platform, @@ -689,6 +699,7 @@ def get_package_finder( def shim_unpack( unpack_fn, # type: TShimmedFunc download_dir, # type str + tempdir_manager_provider, # type: TShimmedFunc ireq=None, # type: Optional[Any] link=None, # type: Optional[Any] location=None, # type Optional[str], @@ -708,6 +719,8 @@ def shim_unpack( :param unpack_fn: A callable or shim referring to the pip implementation :type unpack_fn: Callable :param str download_dir: The directory to download the file to + :param TShimmedFunc tempdir_manager_provider: A callable or shim referring to + `global_tempdir_manager` function from pip or a shimmed no-op context manager :param Optional[:class:`~pip._internal.req.req_install.InstallRequirement`] ireq: an Install Requirement instance, defaults to None :param Optional[:class:`~pip._internal.models.link.Link`] link: A Link instance, @@ -727,31 +740,33 @@ def shim_unpack( """ unpack_fn = resolve_possible_shim(unpack_fn) downloader_provider = resolve_possible_shim(downloader_provider) + tempdir_manager_provider = resolve_possible_shim(tempdir_manager_provider) required_args = inspect.getargs(unpack_fn.__code__).args # type: ignore unpack_kwargs = {"download_dir": download_dir} - if ireq: - if not link and ireq.link: - link = ireq.link - if only_download is None: - only_download = ireq.is_wheel - if hashes is None: - hashes = ireq.hashes(True) - if location is None and getattr(ireq, "source_dir", None): - location = ireq.source_dir - unpack_kwargs.update({"link": link, "location": location}) - if hashes is not None and "hashes" in required_args: - unpack_kwargs["hashes"] = hashes - if "progress_bar" in required_args: - unpack_kwargs["progress_bar"] = progress_bar - if only_download is not None and "only_download" in required_args: - unpack_kwargs["only_download"] = only_download - if session is not None and "session" in required_args: - unpack_kwargs["session"] = session - if "downloader" in required_args and downloader_provider is not None: - assert session is not None - assert progress_bar is not None - unpack_kwargs["downloader"] = downloader_provider(session, progress_bar) - return unpack_fn(**unpack_kwargs) # type: ignore + with tempdir_manager_provider(): + if ireq: + if not link and ireq.link: + link = ireq.link + if only_download is None: + only_download = ireq.is_wheel + if hashes is None: + hashes = ireq.hashes(True) + if location is None and getattr(ireq, "source_dir", None): + location = ireq.source_dir + unpack_kwargs.update({"link": link, "location": location}) + if hashes is not None and "hashes" in required_args: + unpack_kwargs["hashes"] = hashes + if "progress_bar" in required_args: + unpack_kwargs["progress_bar"] = progress_bar + if only_download is not None and "only_download" in required_args: + unpack_kwargs["only_download"] = only_download + if session is not None and "session" in required_args: + unpack_kwargs["session"] = session + if "downloader" in required_args and downloader_provider is not None: + assert session is not None + assert progress_bar is not None + unpack_kwargs["downloader"] = downloader_provider(session, progress_bar) + return unpack_fn(**unpack_kwargs) # type: ignore def _ensure_finder( @@ -860,7 +875,7 @@ def make_preparer( preparer_fn = resolve_possible_shim(preparer_fn) downloader_provider = resolve_possible_shim(downloader_provider) finder_provider = resolve_possible_shim(finder_provider) - required_args = inspect.getargs(preparer_fn.__init__.__code__).args # type: ignore + required_args, required_kwargs = get_allowed_args(preparer_fn) # type: ignore if not req_tracker and not req_tracker_fn and "req_tracker" in required_args: raise TypeError("No requirement tracker and no req tracker generator found!") if "downloader" in required_args and not downloader_provider: @@ -918,6 +933,38 @@ def make_preparer( yield result +@contextlib.contextmanager +def _ensure_wheel_cache( + wheel_cache=None, # type: Optional[Type[TWheelCache]] + wheel_cache_provider=None, # type: Optional[Callable] + format_control=None, # type: Optional[TFormatControl] + format_control_provider=None, # type: Optional[Type[TShimmedFunc]] + options=None, # type: Optional[Values] + cache_dir=None, # type: Optional[str] +): + if wheel_cache is not None: + yield wheel_cache + elif wheel_cache_provider is not None: + with ExitStack() as stack: + cache_dir = getattr(options, "cache_dir", cache_dir) + format_control = getattr( + options, + "format_control", + format_control_provider(None, None), # TFormatControl + ) + wheel_cache = stack.enter_context( + wheel_cache_provider(cache_dir, format_control) + ) + yield wheel_cache + + +def get_ireq_output_path(wheel_cache, ireq): + if getattr(wheel_cache, "get_path_for_link", None): + return wheel_cache.get_path_for_link(ireq.link) + elif getattr(wheel_cache, "cached_wheel", None): + return wheel_cache.cached_wheel(ireq.link, ireq.name).url_without_fragment + + def get_resolver( resolver_fn, # type: TShimmedFunc install_req_provider=None, # type: Optional[TShimmedFunc] @@ -938,6 +985,7 @@ def get_resolver( make_install_req=None, # type: Optional[Callable] install_cmd_provider=None, # type: Optional[TShimmedFunc] install_cmd=None, # type: Optional[TCommandInstance] + use_pep517=True, # type: bool ): # (...) -> TResolver """ @@ -985,6 +1033,7 @@ def get_resolver( to the resolver for actually generating install requirements, if necessary :param Optional[TCommandInstance] install_cmd: The install command used to create the finder, session, and options if needed, defaults to None. + :param bool use_pep517: Whether to use the pep517 build process. :return: A new resolver instance. :rtype: :class:`~pip._internal.legacy_resolve.Resolver` @@ -1049,7 +1098,7 @@ def get_resolver( continue elif val is None and install_cmd is None: raise TypeError( - "Preparer requires a {0} but did not receive one " + "Preparer requires a {} but did not receive one " "and cannot generate one".format(arg) ) elif arg == "session" and val is None: @@ -1059,26 +1108,21 @@ def get_resolver( resolver_kwargs[arg] = val if "make_install_req" in required_args: if make_install_req is None and install_req_provider is not None: + make_install_req_kwargs = { + "isolated": isolated, + "wheel_cache": wheel_cache, + "use_pep517": use_pep517, + } + factory_args, factory_kwargs = filter_allowed_args( + install_req_provider, **make_install_req_kwargs + ) make_install_req = functools.partial( - install_req_provider, - isolated=isolated, - wheel_cache=wheel_cache, - # use_pep517=use_pep517, + install_req_provider, *factory_args, **factory_kwargs ) assert make_install_req is not None resolver_kwargs["make_install_req"] = make_install_req if "isolated" in required_args: resolver_kwargs["isolated"] = isolated - if "wheel_cache" in required_args: - if wheel_cache is None and wheel_cache_provider is not None: - cache_dir = getattr(options, "cache_dir", None) - format_control = getattr( - options, - "format_control", - format_control_provider(None, None), # type: ignore - ) - wheel_cache = wheel_cache_provider(cache_dir, format_control) - resolver_kwargs["wheel_cache"] = wheel_cache resolver_kwargs.update( { "upgrade_strategy": upgrade_strategy, @@ -1090,6 +1134,15 @@ def get_resolver( "preparer": preparer, } ) + if "wheel_cache" in required_args: + with _ensure_wheel_cache( + wheel_cache=wheel_cache, + wheel_cache_provider=wheel_cache_provider, + format_control_provider=format_control_provider, + options=options, + ) as wheel_cache: + resolver_kwargs["wheel_cache"] = wheel_cache + return resolver_fn(**resolver_kwargs) # type: ignore return resolver_fn(**resolver_kwargs) # type: ignore @@ -1255,22 +1308,29 @@ def resolve( # noqa:C901 if session is None: session = get_session(install_cmd=install_command, options=options) if finder is None: - finder = finder_provider(install_command, options=options, session=session) # type: ignore + finder = finder_provider( + install_command, options=options, session=session + ) # type: ignore format_control = getattr(options, "format_control", None) if not format_control: format_control = format_control_provider(None, None) # type: ignore - wheel_cache = wheel_cache_provider( - kwargs["cache_dir"], format_control + wheel_cache = ctx.enter_context( + wheel_cache_provider(kwargs["cache_dir"], format_control) ) # type: ignore ireq.is_direct = True # type: ignore build_location_kwargs = {"build_dir": kwargs["build_dir"], "autodelete": True} call_function_with_correct_args(ireq.build_location, **build_location_kwargs) - # ireq.build_location(kwargs["build_dir"]) # type: ignore if reqset_provider is None: raise TypeError( "cannot resolve without a requirement set provider... failed!" ) - reqset = reqset_provider(install_command, options=options, session=session, wheel_download_dir=wheel_download_dir, **kwargs) # type: ignore + reqset = reqset_provider( + install_command, + options=options, + session=session, + wheel_download_dir=wheel_download_dir, + **kwargs + ) # type: ignore if getattr(reqset, "prepare_files", None): reqset.add_requirement(ireq) results = reqset.prepare_files(finder) @@ -1293,7 +1353,6 @@ def resolve( # noqa:C901 "use_user_site": use_user_site, "require_hashes": require_hashes, } - # with req_tracker_provider() as req_tracker: if isinstance(req_tracker_provider, (types.FunctionType, functools.partial)): preparer_args["req_tracker"] = ctx.enter_context(req_tracker_provider()) resolver_keys = [ @@ -1338,7 +1397,7 @@ def resolve( # noqa:C901 return results -def build_wheel( +def build_wheel( # noqa:C901 req=None, # type: Optional[TInstallRequirement] reqset=None, # type: Optional[Union[TReqSet, Iterable[TInstallRequirement]]] output_dir=None, # type: Optional[str] @@ -1368,8 +1427,9 @@ def build_wheel( build_many_provider=None, # type: Optional[TShimmedFunc] install_command_provider=None, # type: Optional[TShimmedFunc] finder_provider=None, # type: Optional[TShimmedFunc] + reqset_provider=None, # type: Optional[TShimmedFunc] ): - # type: (...) -> Optional[Union[str, Tuple[List[TInstallRequirement], List[TInstallRequirement]]]] + # type: (...) -> Generator[Union[str, Tuple[List[TInstallRequirement], ...]], None, None] """ Build a wheel or a set of wheels @@ -1421,19 +1481,21 @@ def build_wheel( :param Optional[TShimmedFunc] install_command_provider: A shim for providing new install command instances :param TShimmedFunc finder_provider: A provider to package finder instances + :param TShimmedFunc reqset_provider: A provider for requirement set generation :return: A tuple of successful and failed install requirements or else a path to a wheel :rtype: Optional[Union[str, Tuple[List[TInstallRequirement], List[TInstallRequirement]]]] """ wheel_cache_provider = resolve_possible_shim(wheel_cache_provider) - preparer = resolve_possible_shim(preparer) + preparer_provider = resolve_possible_shim(preparer_provider) wheel_builder_provider = resolve_possible_shim(wheel_builder_provider) build_one_provider = resolve_possible_shim(build_one_provider) build_one_inside_env_provider = resolve_possible_shim(build_one_inside_env_provider) build_many_provider = resolve_possible_shim(build_many_provider) install_cmd_provider = resolve_possible_shim(install_command_provider) format_control_provider = resolve_possible_shim(format_control_provider) - finder_provider = resolve_possible_shim(finder_provider) + finder_provider = resolve_possible_shim(finder_provider) or get_package_finder + reqset_provider = resolve_possible_shim(reqset_provider) global_options = [] if global_options is None else global_options build_options = [] if build_options is None else build_options options = None @@ -1447,25 +1509,31 @@ def build_wheel( } if not req and not reqset: raise TypeError("Must provide either a requirement or requirement set to build") - if wheel_cache is None and (reqset is not None or output_dir is None): - if install_command is None: - assert isinstance(install_cmd_provider, (type, functools.partial)) - install_command = install_cmd_provider() - kwargs, options = populate_options(install_command, options, **kwarg_map) - format_control = getattr(options, "format_control", None) - if not format_control: - format_control = format_control_provider(None, None) # type: ignore - wheel_cache = wheel_cache_provider(options.cache_dir, format_control) - if req and not reqset and not output_dir: - output_dir = wheel_cache.get_path_for_link(req.link) - if not reqset and build_one_provider: - yield build_one_provider(req, output_dir, build_options, global_options) - elif build_many_provider: - yield build_many_provider( - reqset, wheel_cache, build_options, global_options, check_binary_allowed - ) - else: - with ExitStack() as ctx: + with ExitStack() as ctx: + kwargs = kwarg_map.copy() + if wheel_cache is None and (reqset is not None or output_dir is None): + if install_command is None: + assert isinstance(install_cmd_provider, (type, functools.partial)) + install_command = install_cmd_provider() + kwargs, options = populate_options(install_command, options, **kwarg_map) + format_control = getattr(options, "format_control", None) + if not format_control: + format_control = format_control_provider(None, None) # type: ignore + wheel_cache = ctx.enter_context( + wheel_cache_provider(options.cache_dir, format_control) + ) + if req and not reqset and not output_dir: + output_dir = get_ireq_output_path(wheel_cache, req) + if not reqset and build_one_provider: + yield build_one_provider(req, output_dir, build_options, global_options) + elif build_many_provider: + yield build_many_provider( + reqset, wheel_cache, build_options, global_options, check_binary_allowed + ) + else: + builder_args, builder_kwargs = get_allowed_args(wheel_builder_provider) + if "requirement_set" in builder_args and not reqset: + reqset = reqset_provider() if session is None and finder is None: session = get_session(install_cmd=install_command, options=options) finder = finder_provider( @@ -1503,7 +1571,7 @@ def build_wheel( ) if req and not reqset: if not output_dir: - output_dir = wheel_cache.get_path_for_link(req.link) + output_dir = get_ireq_output_path(wheel_cache, req) if use_pep517 is not None: req.use_pep517 = use_pep517 yield builder._build_one(req, output_dir) diff --git a/pipenv/vendor/pip_shims/models.py b/pipenv/vendor/pip_shims/models.py index d641c6dd..79871f5b 100644 --- a/pipenv/vendor/pip_shims/models.py +++ b/pipenv/vendor/pip_shims/models.py @@ -104,6 +104,9 @@ PIP_VERSION_SET = { "19.2.3", "19.3", "19.3.1", + "20.0", + "20.0.1", + "20.0.2", } @@ -140,9 +143,9 @@ class PipVersion(Sequence): parsed_version = self._parse() if base_import_path is None: if parsed_version >= parse_version("10.0.0"): - base_import_path = "{0}._internal".format(BASE_IMPORT_PATH) + base_import_path = "{}._internal".format(BASE_IMPORT_PATH) else: - base_import_path = "{0}".format(BASE_IMPORT_PATH) + base_import_path = "{}".format(BASE_IMPORT_PATH) self.base_import_path = base_import_path self.parsed_version = parsed_version @@ -175,13 +178,12 @@ class PipVersion(Sequence): def __str__(self): # type: () -> str - return "{0!s}".format(self.parsed_version) + return "{!s}".format(self.parsed_version) def __repr__(self): # type: () -> str return ( - "" + "" ).format( self.version, self.base_import_path, @@ -253,17 +255,17 @@ class PipVersionRange(Sequence): def __str__(self): # type: () -> str - return "{0!s} -> {1!s}".format(self._versions[0], self._versions[-1]) + return "{!s} -> {!s}".format(self._versions[0], self._versions[-1]) @property def base_import_paths(self): # type: () -> Set[str] - return set([version.base_import_path for version in self._versions]) + return {version.base_import_path for version in self._versions} @property def vendor_import_paths(self): # type: () -> Set[str] - return set([version.vendor_import_path for version in self._versions]) + return {version.vendor_import_path for version in self._versions} def is_valid(self): # type: () -> bool @@ -430,7 +432,7 @@ class ShimmedPath(object): if not self.is_class: return provided if not inspect.isclass(provided): - raise TypeError("Provided argument is not a class: {0!r}".format(provided)) + raise TypeError("Provided argument is not a class: {!r}".format(provided)) methods = self._parse_provides_dict( self.provided_methods, prepend_arg_to_callables="self" ) @@ -554,9 +556,7 @@ class ShimmedPath(object): imported, result = self._shim_parent(imported, attribute_name) if result is not None: result = self._ensure_functions(result) - full_import_path = "{0}.{1}".format( - self.calculated_module_path, attribute_name - ) + full_import_path = "{}.{}".format(self.calculated_module_path, attribute_name) self._imported = imported assert isinstance(result, types.ModuleType) self._provided = result @@ -921,13 +921,6 @@ unpack_url = ShimmedPathCollection("unpack_url", ImportTypes.FUNCTION) unpack_url.create_path("download.unpack_url", "7.0.0", "19.3.9") unpack_url.create_path("operations.prepare.unpack_url", "20.0", "9999") -shim_unpack = ShimmedPathCollection("shim_unpack", ImportTypes.FUNCTION) -shim_unpack.set_default( - functools.partial( - compat.shim_unpack, unpack_fn=unpack_url, downloader_provider=Downloader - ) -) - is_installable_dir = ShimmedPathCollection("is_installable_dir", ImportTypes.FUNCTION) is_installable_dir.create_path("utils.misc.is_installable_dir", "10.0.0", "9999") is_installable_dir.create_path("utils.is_installable_dir", "7.0.0", "9.0.3") @@ -1036,6 +1029,16 @@ global_tempdir_manager.create_path( "utils.temp_dir.global_tempdir_manager", "7.0.0", "9999" ) +shim_unpack = ShimmedPathCollection("shim_unpack", ImportTypes.FUNCTION) +shim_unpack.set_default( + functools.partial( + compat.shim_unpack, + unpack_fn=unpack_url, + downloader_provider=Downloader, + tempdir_manager_provider=global_tempdir_manager, + ) +) + get_requirement_tracker = ShimmedPathCollection( "get_requirement_tracker", ImportTypes.CONTEXTMANAGER ) @@ -1128,6 +1131,17 @@ DEV_PKGS.create_path("commands.freeze.DEV_PKGS", "9.0.0", "9999") DEV_PKGS.set_default({"setuptools", "pip", "distribute", "wheel"}) +wheel_cache = ShimmedPathCollection("wheel_cache", ImportTypes.FUNCTION) +wheel_cache.set_default( + functools.partial( + compat.wheel_cache, + wheel_cache_provider=WheelCache, + tempdir_manager_provider=global_tempdir_manager, + format_control_provider=FormatControl, + ) +) + + get_package_finder = ShimmedPathCollection("get_package_finder", ImportTypes.FUNCTION) get_package_finder.set_default( functools.partial( @@ -1158,7 +1172,7 @@ get_resolver.set_default( install_cmd_provider=InstallCommand, resolver_fn=Resolver, install_req_provider=install_req_from_req_string, - wheel_cache_provider=WheelCache, + wheel_cache_provider=wheel_cache, format_control_provider=FormatControl, ) ) @@ -1170,6 +1184,7 @@ get_requirement_set.set_default( compat.get_requirement_set, install_cmd_provider=InstallCommand, req_set_provider=RequirementSet, + wheel_cache_provider=wheel_cache, ) ) @@ -1182,7 +1197,7 @@ resolve.set_default( reqset_provider=get_requirement_set, finder_provider=get_package_finder, resolver_provider=get_resolver, - wheel_cache_provider=WheelCache, + wheel_cache_provider=wheel_cache, format_control_provider=FormatControl, make_preparer_provider=make_preparer, req_tracker_provider=get_requirement_tracker, @@ -1196,12 +1211,13 @@ build_wheel.set_default( functools.partial( compat.build_wheel, install_command_provider=InstallCommand, - wheel_cache_provider=WheelCache, + wheel_cache_provider=wheel_cache, wheel_builder_provider=WheelBuilder, build_one_provider=build_one, build_one_inside_env_provider=build_one_inside_env, build_many_provider=build, preparer_provider=make_preparer, format_control_provider=FormatControl, + reqset_provider=get_requirement_set, ) ) diff --git a/pipenv/vendor/pip_shims/utils.py b/pipenv/vendor/pip_shims/utils.py index 76661ecc..162b4a20 100644 --- a/pipenv/vendor/pip_shims/utils.py +++ b/pipenv/vendor/pip_shims/utils.py @@ -440,3 +440,26 @@ def call_function_with_correct_args(fn, **provided_kwargs): continue kwargs[arg] = provided_kwargs[arg] return fn(*args, **kwargs) + + +def filter_allowed_args(fn, **provided_kwargs): + # type: (Callable, Dict[str, Any]) -> Tuple[List[Any], Dict[str, Any]] + """ + Given a function and a kwarg mapping, return only those kwargs used in the function. + + :param Callable fn: A function to inspect + :param Dict[str, Any] kwargs: A mapping of kwargs to filter + :return: A new, filtered kwarg mapping + :rtype: Tuple[List[Any], Dict[str, Any]] + """ + args = [] + kwargs = {} + func_args, func_kwargs = get_allowed_args(fn) + for arg in func_args: + if arg in provided_kwargs: + args.append(provided_kwargs[arg]) + for arg in func_kwargs: + if arg not in provided_kwargs: + continue + kwargs[arg] = provided_kwargs[arg] + return args, kwargs diff --git a/pipenv/vendor/pyparsing.py b/pipenv/vendor/pyparsing.py index 4d2f98e4..581d5bbb 100644 --- a/pipenv/vendor/pyparsing.py +++ b/pipenv/vendor/pyparsing.py @@ -95,8 +95,8 @@ classes inherit from. Use the docstrings for examples of how to: namespace class """ -__version__ = "2.4.6" -__versionTime__ = "24 Dec 2019 04:27 UTC" +__version__ = "2.4.7" +__versionTime__ = "30 Mar 2020 00:43 UTC" __author__ = "Paul McGuire " import string @@ -1391,6 +1391,12 @@ class ParserElement(object): """ ParserElement._literalStringClass = cls + @classmethod + def _trim_traceback(cls, tb): + while tb.tb_next: + tb = tb.tb_next + return tb + def __init__(self, savelist=False): self.parseAction = list() self.failAction = None @@ -1943,7 +1949,9 @@ class ParserElement(object): if ParserElement.verbose_stacktrace: raise else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + if getattr(exc, '__traceback__', None) is not None: + exc.__traceback__ = self._trim_traceback(exc.__traceback__) raise exc else: return tokens @@ -2017,7 +2025,9 @@ class ParserElement(object): if ParserElement.verbose_stacktrace: raise else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + if getattr(exc, '__traceback__', None) is not None: + exc.__traceback__ = self._trim_traceback(exc.__traceback__) raise exc def transformString(self, instring): @@ -2063,7 +2073,9 @@ class ParserElement(object): if ParserElement.verbose_stacktrace: raise else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + if getattr(exc, '__traceback__', None) is not None: + exc.__traceback__ = self._trim_traceback(exc.__traceback__) raise exc def searchString(self, instring, maxMatches=_MAX_INT): @@ -2093,7 +2105,9 @@ class ParserElement(object): if ParserElement.verbose_stacktrace: raise else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + if getattr(exc, '__traceback__', None) is not None: + exc.__traceback__ = self._trim_traceback(exc.__traceback__) raise exc def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False): @@ -2565,7 +2579,9 @@ class ParserElement(object): if ParserElement.verbose_stacktrace: raise else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + if getattr(exc, '__traceback__', None) is not None: + exc.__traceback__ = self._trim_traceback(exc.__traceback__) raise exc def __eq__(self, other): @@ -2724,7 +2740,7 @@ class ParserElement(object): continue if not t: continue - out = ['\n'.join(comments), t] + out = ['\n' + '\n'.join(comments) if comments else '', t] comments = [] try: # convert newline marks to actual newlines, and strip leading BOM if present @@ -3312,7 +3328,7 @@ class Regex(Token): self.name = _ustr(self) self.errmsg = "Expected " + self.name self.mayIndexError = False - self.mayReturnEmpty = True + self.mayReturnEmpty = self.re_match("") is not None self.asGroupList = asGroupList self.asMatch = asMatch if self.asGroupList: @@ -3993,6 +4009,7 @@ class And(ParseExpression): self.leaveWhitespace() def __init__(self, exprs, savelist=True): + exprs = list(exprs) if exprs and Ellipsis in exprs: tmp = [] for i, expr in enumerate(exprs): @@ -4358,7 +4375,7 @@ class Each(ParseExpression): if self.initExprGroups: self.opt1map = dict((id(e.expr), e) for e in self.exprs if isinstance(e, Optional)) opt1 = [e.expr for e in self.exprs if isinstance(e, Optional)] - opt2 = [e for e in self.exprs if e.mayReturnEmpty and not isinstance(e, Optional)] + opt2 = [e for e in self.exprs if e.mayReturnEmpty and not isinstance(e, (Optional, Regex))] self.optionals = opt1 + opt2 self.multioptionals = [e.expr for e in self.exprs if isinstance(e, ZeroOrMore)] self.multirequired = [e.expr for e in self.exprs if isinstance(e, OneOrMore)] @@ -5435,8 +5452,8 @@ def matchPreviousExpr(expr): return rep def _escapeRegexRangeChars(s): - # ~ escape these chars: ^-] - for c in r"\^-]": + # ~ escape these chars: ^-[] + for c in r"\^-[]": s = s.replace(c, _bslash + c) s = s.replace("\n", r"\n") s = s.replace("\t", r"\t") @@ -6550,10 +6567,10 @@ class pyparsing_common: """mixed integer of the form 'integer - fraction', with optional leading integer, returns float""" mixed_integer.addParseAction(sum) - real = Regex(r'[+-]?(:?\d+\.\d*|\.\d+)').setName("real number").setParseAction(convertToFloat) + real = Regex(r'[+-]?(?:\d+\.\d*|\.\d+)').setName("real number").setParseAction(convertToFloat) """expression that parses a floating point number and returns a float""" - sci_real = Regex(r'[+-]?(:?\d+(:?[eE][+-]?\d+)|(:?\d+\.\d*|\.\d+)(:?[eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) + sci_real = Regex(r'[+-]?(?:\d+(?:[eE][+-]?\d+)|(?:\d+\.\d*|\.\d+)(?:[eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) """expression that parses a floating point number with optional scientific notation and returns a float""" diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 488da849..b8270bb7 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -10,7 +10,7 @@ from .models.lockfile import Lockfile from .models.pipfile import Pipfile from .models.requirements import Requirement -__version__ = "1.5.4" +__version__ = "1.5.7" logger = logging.getLogger(__name__) diff --git a/pipenv/vendor/requirementslib/models/markers.py b/pipenv/vendor/requirementslib/models/markers.py index 8dda0495..84637642 100644 --- a/pipenv/vendor/requirementslib/models/markers.py +++ b/pipenv/vendor/requirementslib/models/markers.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import itertools import operator +import re import attr import distlib.markers @@ -196,15 +197,19 @@ def _get_specs(specset): return sorted(result, key=operator.itemgetter(1)) +# TODO: Rename this to something meaningful def _group_by_op(specs): # type: (Union[Set[Specifier], SpecifierSet]) -> Iterator specs = [_get_specs(x) for x in list(specs)] - flattened = [(op, version) for spec in specs for op, version in spec] + flattened = [ + ((op, len(version) > 2), version) for spec in specs for op, version in spec + ] specs = sorted(flattened) grouping = itertools.groupby(specs, key=operator.itemgetter(0)) return grouping +# TODO: rename this to something meaningful def normalize_specifier_set(specs): # type: (Union[str, SpecifierSet]) -> Optional[Set[Specifier]] """Given a specifier set, a string, or an iterable, normalize the specifiers @@ -235,6 +240,8 @@ def normalize_specifier_set(specs): return normalize_specifier_set(SpecifierSet(",".join(spec_list))) +# TODO: Check if this is used by anything public otherwise make it private +# And rename it to something meaningful def get_sorted_version_string(version_set): # type: (Set[AnyStr]) -> AnyStr version_list = sorted( @@ -244,6 +251,9 @@ def get_sorted_version_string(version_set): return version +# TODO: Rename this to something meaningful +# TODO: Add a deprecation decorator and deprecate this -- i'm sure it's used +# in other libraries @lru_cache(maxsize=1024) def cleanup_pyspecs(specs, joiner="or"): specs = normalize_specifier_set(specs) @@ -275,7 +285,8 @@ def cleanup_pyspecs(specs, joiner="or"): "==": lambda x: "in" if len(x) > 1 else "==", } translation_keys = list(translation_map.keys()) - for op, versions in _group_by_op(tuple(specs)): + for op_and_version_type, versions in _group_by_op(tuple(specs)): + op = op_and_version_type[0] versions = [version[1] for version in versions] versions = sorted(dedup(versions)) op_key = next(iter(k for k in translation_keys if op in k), None) @@ -284,10 +295,11 @@ def cleanup_pyspecs(specs, joiner="or"): version_value = translation_map[op_key][joiner](versions) if op in op_translations: op = op_translations[op](versions) - results[op] = version_value - return sorted([(k, v) for k, v in results.items()], key=operator.itemgetter(1)) + results[(op, op_and_version_type[1])] = version_value + return sorted([(k[0], v) for k, v in results.items()], key=operator.itemgetter(1)) +# TODO: Rename this to something meaningful @lru_cache(maxsize=1024) def fix_version_tuple(version_tuple): # type: (Tuple[AnyStr, AnyStr]) -> Tuple[AnyStr, AnyStr] @@ -302,6 +314,7 @@ def fix_version_tuple(version_tuple): return (op, version) +# TODO: Rename this to something meaningful, deprecate it (See prior function) @lru_cache(maxsize=128) def get_versions(specset, group_by_operator=True): # type: (Union[Set[Specifier], SpecifierSet], bool) -> List[Tuple[STRING_TYPE, STRING_TYPE]] @@ -590,6 +603,7 @@ def get_specset(marker_list): return specifiers +# TODO: Refactor this (reduce complexity) def parse_marker_dict(marker_dict): op = marker_dict["op"] lhs = marker_dict["lhs"] @@ -658,9 +672,16 @@ def parse_marker_dict(marker_dict): return specset, finalized_marker +def _contains_micro_version(version_string): + return re.search("\d+\.\d+\.\d+", version_string) is not None + + def format_pyversion(parts): op, val = parts - return "python_version {0} '{1}'".format(op, val) + version_marker = ( + "python_full_version" if _contains_micro_version(val) else "python_version" + ) + return "{0} {1} '{2}'".format(version_marker, op, val) def normalize_marker_str(marker): @@ -699,3 +720,16 @@ def marker_from_specifier(spec): marker_segments.append(format_pyversion(marker_segment)) marker_str = " and ".join(marker_segments).replace('"', "'") return Marker(marker_str) + + +def merge_markers(m1, m2): + # type: (Marker, Marker) -> Optional[Marker] + if not all((m1, m2)): + return next(iter(v for v in (m1, m2) if v), None) + m1 = _ensure_marker(m1) + m2 = _ensure_marker(m2) + _markers = [] # type: List[Marker] + for marker in (m1, m2): + _markers.append(str(marker)) + marker_str = " and ".join([normalize_marker_str(m) for m in _markers if m]) + return _ensure_marker(normalize_marker_str(marker_str)) diff --git a/pipenv/vendor/requirementslib/models/metadata.py b/pipenv/vendor/requirementslib/models/metadata.py new file mode 100644 index 00000000..912b1b77 --- /dev/null +++ b/pipenv/vendor/requirementslib/models/metadata.py @@ -0,0 +1,1240 @@ +# -*- coding=utf-8 -*- +import datetime +import functools +import io +import json +import logging +import operator +import os +import zipfile +from collections import defaultdict + +import attr +import dateutil.parser +import distlib.metadata +import distlib.wheel +import packaging.version +import requests +import six +import vistir +from packaging.markers import Marker +from packaging.requirements import Requirement as PackagingRequirement +from packaging.specifiers import Specifier, SpecifierSet +from packaging.tags import Tag + +from ..environment import MYPY_RUNNING +from .markers import ( + get_contained_extras, + get_contained_pyversions, + get_without_extra, + get_without_pyversion, + marker_from_specifier, + merge_markers, + normalize_specifier_set, +) +from .requirements import Requirement +from .utils import filter_dict, get_pinned_version, is_pinned_requirement + +# fmt: off +from six.moves import Sequence # type: ignore # isort:skip +from six.moves import reduce # type: ignore # isort:skip +# fmt: on # isort:skip + + +ch = logging.StreamHandler() +formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s") +ch.setFormatter(formatter) +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +if MYPY_RUNNING: + from typing import ( + Any, + Callable, + Dict, + Generator, + Generic, + Iterator, + List, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, + ) + from attr import Attribute # noqa + from .setup_info import SetupInfo + + TAttrsClass = TypeVar("TAttrsClass") + AttrsClass = Generic[TAttrsClass] + TDigestDict = Dict[str, str] + TProjectUrls = Dict[str, str] + TReleaseUrlDict = Dict[str, Union[bool, int, str, TDigestDict]] + TReleasesList = List[TReleaseUrlDict] + TReleasesDict = Dict[str, TReleasesList] + TDownloads = Dict[str, int] + TPackageInfo = Dict[str, Optional[Union[str, List[str], TDownloads, TProjectUrls]]] + TPackage = Dict[str, Union[TPackageInfo, int, TReleasesDict, TReleasesList]] + + +VALID_ALGORITHMS = { + "sha1": 40, + "sha3_224": 56, + "sha512": 128, + "blake2b": 128, + "sha256": 64, + "sha384": 96, + "blake2s": 64, + "sha3_256": 64, + "sha3_512": 128, + "md5": 32, + "sha3_384": 96, + "sha224": 56, +} # type: Dict[str, int] + +PACKAGE_TYPES = { + "sdist", + "bdist_wheel", + "bdist_egg", + "bdist_dumb", + "bdist_wininst", + "bdist_rpm", + "bdist_msi", + "bdist_dmg", +} + + +class PackageEncoder(json.JSONEncoder): + def default(self, obj): # noqa:E0202 # noqa:W0221 + if isinstance(obj, datetime.datetime): + return obj.isoformat() + elif isinstance(obj, PackagingRequirement): + return obj.__dict__ + elif isinstance(obj, set): + return tuple(obj) + elif isinstance(obj, (Specifier, SpecifierSet, Marker)): + return str(obj) + else: + return json.JSONEncoder.default(self, obj) + + +def validate_extras(inst, attrib, value): + # type: ("Dependency", Attribute, Tuple[str, ...]) -> None + duplicates = [k for k in value if value.count(k) > 1] + if duplicates: + raise ValueError("Found duplicate keys: {0}".format(", ".join(duplicates))) + return None + + +def validate_digest(inst, attrib, value): + # type: ("Digest", Attribute, str) -> None + expected_length = VALID_ALGORITHMS[inst.algorithm.lower()] + if len(value) != expected_length: + raise ValueError( + "Expected a digest of length {0!s}, got one of length {1!s}".format( + expected_length, len(value) + ) + ) + return None + + +def get_local_wheel_metadata(wheel_file): + # type: (str) -> Optional[distlib.metadata.Metadata] + parsed_metadata = None + with io.open(wheel_file, "rb") as fh: + with zipfile.ZipFile(fh, mode="r", compression=zipfile.ZIP_DEFLATED) as zf: + metadata = None + for fn in zf.namelist(): + if os.path.basename(fn) == "METADATA": + metadata = fn + break + if metadata is None: + raise RuntimeError("No metadata found in wheel: {0}".format(wheel_file)) + with zf.open(metadata, "r") as metadata_fh: + parsed_metadata = distlib.metadata.Metadata(fileobj=metadata_fh) + return parsed_metadata + + +def get_remote_sdist_metadata(line): + # type: (str) -> SetupInfo + req = Requirement.from_line(line) + try: + _ = req.run_requires() + except SystemExit: + raise RuntimeError("Failed to compute metadata for dependency {0}".format(line)) + else: + return req.line_instance.setup_info + + +def get_remote_wheel_metadata(whl_file): + # type: (str) -> Optional[distlib.metadata.Metadata] + parsed_metadata = None + data = io.BytesIO() + with vistir.contextmanagers.open_file(whl_file) as fp: + for chunk in iter(lambda: fp.read(8096), b""): + data.write(chunk) + with zipfile.ZipFile(data, mode="r", compression=zipfile.ZIP_DEFLATED) as zf: + metadata = None + for fn in zf.namelist(): + if os.path.basename(fn) == "METADATA": + metadata = fn + break + if metadata is None: + raise RuntimeError("No metadata found in wheel: {0}".format(whl_file)) + with zf.open(metadata, "r") as metadata_fh: + parsed_metadata = distlib.metadata.Metadata(fileobj=metadata_fh) + return parsed_metadata + + +def create_specifierset(spec=None): + # type: (Optional[str]) -> SpecifierSet + if isinstance(spec, SpecifierSet): + return spec + elif isinstance(spec, (set, list, tuple)): + spec = " and ".join(spec) + if spec is None: + spec = "" + return SpecifierSet(spec) + + +@attr.s(frozen=True, eq=True) +class ExtrasCollection(object): + #: The name of the extras collection (e.g. 'security') + name = attr.ib(type=str) + #: The dependency the collection belongs to + parent = attr.ib(type="Dependency") + #: The members of the collection + dependencies = attr.ib(factory=set) # type: Set["Dependency"] + + def add_dependency(self, dependency): + # type: ("Dependency") -> "ExtrasCollection" + if not isinstance(dependency, Dependency): + raise TypeError( + "Expected a Dependency instance, received {0!r}".format(dependency) + ) + dependencies = self.dependencies.copy() + dependencies.add(dependency) + return attr.evolve(self, dependencies=dependencies) + + +@attr.s(frozen=True, eq=True) +class Dependency(object): + #: The name of the dependency + name = attr.ib(type=str) + #: A requirement instance + requirement = attr.ib(type=PackagingRequirement, eq=False) + #: The specifier defined in the dependency definition + specifier = attr.ib(type=SpecifierSet, converter=create_specifierset, eq=False) + #: Any extras this dependency declares + extras = attr.ib(factory=tuple, validator=validate_extras) # type: Tuple[str, ...] + #: The name of the extra meta-dependency this one came from (e.g. 'security') + from_extras = attr.ib(default=None, eq=False) # type: Optional[str] + #: The declared specifier set of allowable python versions for this dependency + python_version = attr.ib( + default="", type=SpecifierSet, converter=create_specifierset, eq=False + ) + #: The parent of this dependency (i.e. where it came from) + parent = attr.ib(default=None) # type: Optional[Dependency] + #: The markers for this dependency + markers = attr.ib(default=None, eq=False) # type: Optional[Marker] + _specset_str = attr.ib(default="", type=str) + _python_version_str = attr.ib(default="", type=str) + _marker_str = attr.ib(default="", type=str) + + def __str__(self): + # type: () -> str + return str(self.requirement) + + def as_line(self): + # type: () -> str + line_str = "{0}".format(self.name) + if self.extras: + line_str = "{0}[{1}]".format(line_str, ",".join(self.extras)) + if self.specifier: + line_str = "{0}{1!s}".format(line_str, self.specifier) + py_version_part = "" + if self.python_version: + specifiers = normalize_specifier_set(self.python_version) + markers = [] + if specifiers is not None: + markers = [marker_from_specifier(str(s)) for s in specifiers] + py_version_part = reduce(merge_markers, markers) + if self.markers: + line_str = "{0}; {1}".format(line_str, str(self.markers)) + if py_version_part: + line_str = "{0} and {1}".format(line_str, py_version_part) + elif py_version_part and not self.markers: + line_str = "{0}; {1}".format(line_str, py_version_part) + return line_str + + def pin(self): + # type: () -> "Package" + base_package = get_package(self.name) + sorted_releases = sorted( + base_package.releases.non_yanked_releases, + key=operator.attrgetter("parsed_version"), + reverse=True, + ) + version = next( + iter(self.specifier.filter((r.version for r in sorted_releases))), None + ) + if not version: + version = next( + iter( + self.specifier.filter( + (r.version for r in sorted_releases), prereleases=True + ) + ), + None, + ) + if not version: + raise RuntimeError( + "Failed to resolve {0} ({1!s})".format(self.name, self.specifier) + ) + match = get_package_version(self.name, str(version)) + return match + + @classmethod + def from_requirement(cls, req, parent=None): + # type: (PackagingRequirement, Optional["Dependency"]) -> "Dependency" + from_extras, marker, python_version = None, None, None + specset_str, py_version_str, marker_str = "", "", "" + if req.marker: + marker = Marker(str(req.marker)) + from_extras = next(iter(list(get_contained_extras(marker))), None) + python_version = get_contained_pyversions(marker) + marker = get_without_extra(get_without_pyversion(marker)) + if not str(marker) or not marker or not marker._markers: + marker = None + req.marker = marker + if marker is not None: + marker_str = str(marker) + if req.specifier: + specset_str = str(req.specifier) + if python_version: + py_version_str = str(python_version) + return cls( + name=req.name, + specifier=req.specifier, + extras=tuple(sorted(set(req.extras))) if req.extras is not None else req.extras, + requirement=req, + from_extras=from_extras, + python_version=python_version, + markers=marker, + parent=parent, + specset_str=specset_str, + python_version_str=py_version_str, + marker_str=marker_str, + ) + + @classmethod + def from_info(cls, info): + # type: ("PackageInfo") -> "Dependency" + marker_str = "" + specset_str, py_version_str = "", "" + if info.requires_python: + # XXX: Some markers are improperly formatted -- we already handle most cases + # XXX: but learned about new broken formats, such as + # XXX: python_version in "2.6 2.7 3.2 3.3" (note the lack of commas) + # XXX: as a marker on a dependency of a library called 'pickleshare' + # XXX: Some packages also have invalid markers with stray characters, + # XXX: such as 'algoliasearch' + try: + marker = marker_from_specifier(info.requires_python) + except Exception: + marker_str = "" + else: + if not marker or not marker._markers: + marker_str = "" + else: + marker_str = "{0!s}".format(marker) + req_str = "{0}=={1}".format(info.name, info.version) + if marker_str: + req_str = "{0}; {1}".format(req_str, marker_str) + req = PackagingRequirement(req_str) + requires_python_str = ( + info.requires_python if info.requires_python is not None else "" + ) + if req.specifier: + specset_str = str(req.specifier) + if requires_python_str: + py_version_str = requires_python_str + return cls( + name=info.name, + specifier=req.specifier, + extras=tuple(sorted(set(req.extras))) if req.extras is not None else req.extras, + requirement=req, + from_extras=None, + python_version=SpecifierSet(requires_python_str), + markers=None, + parent=None, + specset_str=specset_str, + python_version_str=py_version_str, + marker_str=marker_str, + ) + + @classmethod + def from_str(cls, depstr, parent=None): + # type: (str, Optional["Dependency"]) -> "Dependency" + try: + req = PackagingRequirement(depstr) + except Exception: + raise + return cls.from_requirement(req, parent=parent) + + def add_parent(self, parent): + # type: ("Dependency") -> "Dependency" + return attr.evolve(self, parent=parent) + + +@attr.s(frozen=True, eq=True) +class Digest(object): + #: The algorithm declared for the digest, e.g. 'sha256' + algorithm = attr.ib( + type=str, validator=attr.validators.in_(VALID_ALGORITHMS.keys()), eq=True + ) + #: The digest value + value = attr.ib(type=str, validator=validate_digest, eq=True) + + def __str__(self): + # type: () -> str + return "{0}:{1}".format(self.algorithm, self.value) + + @classmethod + def create(cls, algorithm, value): + # type: (str, str) -> "Digest" + return cls(algorithm=algorithm, value=value) + + @classmethod + def collection_from_dict(cls, digest_dict): + # type: (TDigestDict) -> List["Digest"] + return [cls.create(k, v) for k, v in digest_dict.items()] + + +# XXX: This is necessary because attrs converters can only be functions, not classmethods +def create_digest_collection(digest_dict): + # type: (TDigestDict) -> List["Digest"] + return Digest.collection_from_dict(digest_dict) + + +def instance_check_converter(expected_type=None, converter=None): + # type: (Optional[Type], Optional[Callable]) -> Callable + def _converter(val): + if expected_type is not None and isinstance(val, expected_type): + return val + return converter(val) + + return _converter + + +@attr.s(frozen=True, eq=True) +class ParsedTag(object): + #: The marker string corresponding to the tag + marker_string = attr.ib(default=None) # type: Optional[str] + #: The python version represented by the tag + python_version = attr.ib(default=None) # type: Optional[str] + #: The platform represented by the tag + platform_system = attr.ib(default=None) # type: Optional[str] + #: the ABI represented by the tag + abi = attr.ib(default=None) # type: Optional[str] + + +def parse_tag(tag): + # type: (Tag) -> ParsedTag + """ + Parse a :class:`~packaging.tags.Tag` instance + + :param :class:`~packaging.tags.Tag` tag: A tag to parse + :return: A parsed tag with combined markers, supported platform and python version + :rtype: :class:`~ParsedTag` + """ + platform_system = None + python_version = None + version = None + marker_str = "" + if tag.platform.startswith("macos"): + platform_system = "Darwin" + elif tag.platform.startswith("manylinux") or tag.platform.startswith("linux"): + platform_system = "Linux" + elif tag.platform.startswith("win32"): + platform_system = "Windows" + if platform_system: + marker_str = 'platform_system == "{}"'.format(platform_system) + if tag.interpreter: + version = tag.interpreter[2:] + py_version_str = "" + if len(version) == 1: + py_version_str = ">={}.0,<{}".format(version, str(int(version) + 1)) + elif len(version) > 1 and len(version) <= 3: + # reverse the existing version so we can add 1 to the first element + # and re-reverse, generating the new version, e.g. [3, 2, 8] => + # [8, 2, 3] => [9, 2, 3] => [3, 2, 9] + next_version_list = list(reversed(version[:])) + next_version_list[0] = str(int(next_version_list[0]) + 1) + next_version = ".".join(list(reversed(next_version_list))) + version = ".".join(version) + py_version_str = ">={},<{}".format(version, next_version) + else: + py_version_str = "{0}".format(version) + python_version = marker_from_specifier(py_version_str) + if python_version: + if marker_str: + marker_str = "{0} and {1!s}".format(marker_str, python_version) + else: + marker_str = str(python_version) + return ParsedTag( + marker_string=marker_str, + python_version=version, + platform_system=platform_system, + abi=tag.abi, + ) + + +@attr.s(frozen=True, eq=True) +class ReleaseUrl(object): + #: The MD5 digest of the given release + md5_digest = attr.ib(type=Digest) + #: The package type of the url + packagetype = attr.ib(type=str, validator=attr.validators.in_(PACKAGE_TYPES)) + #: The upload timestamp from the package + upload_time = attr.ib( + type=datetime.datetime, + converter=instance_check_converter(datetime.datetime, dateutil.parser.parse), # type: ignore + ) + #: The ISO8601 formatted upload timestamp of the package + upload_time_iso_8601 = attr.ib( + type=datetime.datetime, + converter=instance_check_converter(datetime.datetime, dateutil.parser.parse), # type: ignore + ) + #: The size in bytes of the package + size = attr.ib(type=int) + #: The URL of the package + url = attr.ib(type=str) + #: The digests of the package + digests = attr.ib( + converter=instance_check_converter(list, create_digest_collection) # type: ignore + ) # type: List[Digest] + #: The name of the package + name = attr.ib(type=str, default=None) + #: The available comments of the given upload + comment_text = attr.ib(type=str, default="") + #: The number of downloads (deprecated) + downloads = attr.ib(type=int, default=-1) + #: The filename of the current upload + filename = attr.ib(type=str, default="") + #: Whether the upload has a signature + has_sig = attr.ib(type=bool, default=False) + #: The python_version attribute of the upload (e.g. 'source', 'py27', etc) + python_version = attr.ib(type=str, default="source") + #: The 'requires_python' restriction on the package + requires_python = attr.ib(type=str, default=None) + #: A list of valid aprsed tags from the upload + tags = attr.ib(factory=list) # type: List[ParsedTag] + + @property + def is_wheel(self): + # type: () -> bool + return os.path.splitext(self.filename)[-1].lower() == ".whl" + + @property + def is_sdist(self): + # type: () -> bool + return self.python_version == "source" + + @property + def markers(self): + # type: () -> Optional[str] + # TODO: Compare dependencies in parent and add markers for python version + # TODO: Compare dependencies in parent and add markers for platform + # XXX: We can't use wheel-based markers until we do it via derived markers by + # XXX: comparing in the parent (i.e. 'Release' instance or so) and merging + # XXX: down to the common / minimal set of markers otherwise we wind up + # XXX: with an unmanageable set and combinatorial explosion + # if self.is_wheel: + # return self.get_markers_from_wheel() + if self.requires_python: + return marker_from_specifier(self.requires_python) + return None + + @property + def pep508_url(self): + # type: () -> str + markers = self.markers + req_str = "{0} @ {1}#egg={0}".format(self.name, self.url) + if markers: + req_str = "{0}; {1}".format(req_str, markers) + return req_str + + def get_markers_from_wheel(self): + # type: () -> str + supported_platforms = [] # type: List[str] + supported_pyversions = [] + supported_abis = [] + markers = [] + for parsed_tag in self.tags: + if parsed_tag.marker_string: + markers.append(Marker(parsed_tag.marker_string)) + if parsed_tag.python_version: + supported_pyversions.append(parsed_tag.python_version) + if parsed_tag.abi: + supported_abis.append(parsed_tag.abi) + if not (markers or supported_platforms): + return "" + if ( + all(pyversion in supported_pyversions for pyversion in ["2", "3"]) + and not supported_platforms + ): + marker_line = "" + else: + marker_line = " or ".join(["{}".format(str(marker)) for marker in markers]) + return marker_line + + def get_dependencies(self): + # type: () -> Tuple["ReleaseUrl", Dict[str, Union[List[str], str]]] + results = {"requires_python": None} + requires_dist = [] # type: List[str] + if self.is_wheel: + metadata = get_remote_wheel_metadata(self.url) + if metadata is not None: + requires_dist = metadata.run_requires + if not self.requires_python: + results["requires_python"] = metadata._legacy.get("Requires-Python") + else: + try: + metadata = get_remote_sdist_metadata(self.pep508_url) + except Exception: + requires_dist = [] + else: + requires_dist = [str(v) for v in metadata.requires.values()] + results["requires_dist"] = requires_dist + requires_python = getattr(self, "requires_python", results["requires_python"]) + return attr.evolve(self, requires_python=requires_python), results + + @property + def sha256(self): + # type: () -> str + return next( + iter(digest for digest in self.digests if digest.algorithm == "sha256") + ).value + + @classmethod + def create(cls, release_dict, name=None): + # type: (TReleaseUrlDict, Optional[str]) -> "ReleaseUrl" + valid_digest_keys = set("{0}_digest".format(k) for k in VALID_ALGORITHMS.keys()) + digest_keys = set(release_dict.keys()) & valid_digest_keys + creation_kwargs = {} # type: Dict[str, Union[bool, int, str, Digest, TDigestDict]] + creation_kwargs = { + k: v for k, v in release_dict.items() if k not in digest_keys + } + if name is not None: + creation_kwargs["name"] = name + for k in digest_keys: + digest = release_dict[k] + if not isinstance(digest, six.string_types): + raise TypeError("Digests must be strings, got {!r}".format(digest)) + creation_kwargs[k] = Digest.create(k.replace("_digest", ""), digest) + release_url = cls(**filter_dict(creation_kwargs)) # type: ignore + if release_url.is_wheel: + supported_tags = [ + parse_tag(Tag(*tag)) for tag in distlib.wheel.Wheel(release_url.url).tags + ] + release_url = attr.evolve(release_url, tags=supported_tags) + return release_url + + +def create_release_urls_from_list(urls, name=None): + # type: (Union[TReleasesList, List[ReleaseUrl]], Optional[str]) -> List[ReleaseUrl] + url_list = [] + for release_dict in urls: + if isinstance(release_dict, ReleaseUrl): + if name and not release_dict.name: + release_dict = attr.evolve(release_dict, name=name) + url_list.append(release_dict) + continue + url_list.append(ReleaseUrl.create(release_dict, name=name)) + return url_list + + +@attr.s(frozen=True, eq=True) +class ReleaseUrlCollection(Sequence): + #: A list of release URLs + urls = attr.ib(converter=create_release_urls_from_list) + #: the name of the package + name = attr.ib(default=None) # type: Optional[str] + + @classmethod + def create(cls, urls, name=None): + # type: (TReleasesList, Optional[str]) -> "ReleaseUrlCollection" + return cls(urls=urls, name=name) + + @property + def wheels(self): + # type: () -> Iterator[ReleaseUrl] + for url in self.urls: + if not url.is_wheel: + continue + yield url + + @property + def sdists(self): + # type: () -> Iterator[ReleaseUrl] + for url in self.urls: + if not url.is_sdist: + continue + yield url + + def __iter__(self): + # type: () -> Iterator[ReleaseUrl] + return iter(self.urls) + + def __getitem__(self, key): + # type: (int) -> ReleaseUrl + return self.urls.__getitem__(key) + + def __len__(self): + # type: () -> int + return len(self.urls) + + @property + def latest(self): + # type: () -> Optional[ReleaseUrl] + if not self.urls: + return None + return next( + iter(sorted(self.urls, key=operator.attrgetter("upload_time"), reverse=True)) + ) + + @property + def latest_timestamp(self): + # type: () -> Optional[datetime.datetime] + latest = self.latest + if latest is not None: + return latest.upload_time + return None + + def find_package_type(self, type_): + # type: (str) -> Optional[ReleaseUrl] + """ + Given a package type (e.g. sdist, bdist_wheel), find the matching release + + :param str type_: A package type from :const:`~PACKAGE_TYPES` + :return: The package from this collection matching that type, if available + :rtype: Optional[ReleaseUrl] + """ + if type_ not in PACKAGE_TYPES: + raise ValueError( + "Invalid package type: {0}. Expected one of {1}".format( + type_, " ".join(PACKAGE_TYPES) + ) + ) + return next(iter(url for url in self.urls if url.packagetype == type_), None) + + +def convert_release_urls_to_collection(urls=None, name=None): + # type: (Optional[TReleasesList], Optional[str]) -> ReleaseUrlCollection + if urls is None: + urls = [] + urls = create_release_urls_from_list(urls, name=name) + return ReleaseUrlCollection.create(urls, name=name) + + +@attr.s(frozen=True) +class Release(Sequence): + #: The version of the release + version = attr.ib(type=str) + #: The URL collection for the release + urls = attr.ib( + converter=instance_check_converter( # type: ignore + ReleaseUrlCollection, convert_release_urls_to_collection + ), + type=ReleaseUrlCollection, + ) + #: the name of the package + name = attr.ib(default=None) # type: Optional[str] + + def __iter__(self): + # type: () -> Iterator[ReleaseUrlCollection] + return iter(self.urls) + + def __getitem__(self, key): + return self.urls[key] + + def __len__(self): + # type: () -> int + return len(self.urls) + + @property + def yanked(self): + # type: () -> bool + if not self.urls: + return True + return False + + @property + def parsed_version(self): + # type: () -> packaging.version._BaseVersion + return packaging.version.parse(self.version) + + @property + def wheels(self): + # type: () -> Iterator[ReleaseUrl] + return self.urls.wheels + + @property + def sdists(self): + # type: () -> Iterator[ReleaseUrl] + return self.urls.sdists + + @property + def latest(self): + # type: () -> ReleaseUrl + return self.urls.latest + + @property + def latest_timestamp(self): + # type: () -> datetime.datetime + return self.urls.latest_timestamp + + def to_lockfile(self): + # type: () -> Dict[str, Union[List[str], str]] + return { + "hashes": [str(url.sha256) for url in self.urls if url.sha256 is not None], + "version": "=={0}".format(self.version), + } + + +def get_release(version, urls, name=None): + # type: (str, TReleasesList, Optional[str]) -> Release + release_kwargs = {"version": version, "name": name} + if not isinstance(urls, ReleaseUrlCollection): + release_kwargs["urls"] = convert_release_urls_to_collection(urls, name=name) + else: + release_kwargs["urls"] = urls + return Release(**release_kwargs) # type: ignore + + +def get_releases_from_package(releases, name=None): + # type: (TReleasesDict, Optional[str]) -> List[Release] + release_list = [] + for version, urls in releases.items(): + release_list.append(get_release(version, urls, name=name)) + return release_list + + +@attr.s(frozen=True) +class ReleaseCollection(object): + releases = attr.ib( + factory=list, converter=instance_check_converter(list, get_releases_from_package), # type: ignore + ) # type: List[Release] + + def __iter__(self): + # type: () -> Iterator[Release] + return iter(self.releases) + + def __getitem__(self, key): + # type: (str) -> Release + result = next(iter(r for r in self.releases if r.version == key), None) + if result is None: + raise KeyError(key) + return result + + def __len__(self): + # type: () -> int + return len(self.releases) + + def get_latest_lockfile(self): + # type: () -> Dict[str, Union[str, List[str]]] + return self.latest.to_lockfile() + + def wheels(self): + # type: () -> Iterator[ReleaseUrl] + for release in self.sort_releases(): + for wheel in release.wheels: + yield wheel + + def sdists(self): + # type: () -> Iterator[ReleaseUrl] + for release in self.sort_releases(): + for sdist in release.sdists: + yield sdist + + @property + def non_yanked_releases(self): + # type: () -> List[Release] + return list(r for r in self.releases if not r.yanked) + + def sort_releases(self): + # type: () -> List[Release] + return sorted( + self.non_yanked_releases, + key=operator.attrgetter("latest_timestamp"), + reverse=True, + ) + + @property + def latest(self): + # type: () -> Optional[Release] + return next(iter(r for r in self.sort_releases() if not r.yanked)) + + @classmethod + def load(cls, releases, name=None): + # type: (Union[TReleasesDict, List[Release]], Optional[str]) -> "ReleaseCollection" + if not isinstance(releases, list): + releases = get_releases_from_package(releases, name=name) + return cls(releases) + + +def convert_releases_to_collection(releases, name=None): + # type: (TReleasesDict, Optional[str]) -> ReleaseCollection + return ReleaseCollection.load(releases, name=name) + + +def split_keywords(value): + # type: (Union[str, List]) -> List[str] + if value and isinstance(value, six.string_types): + return value.split(",") + elif isinstance(value, list): + return value + return [] + + +def create_dependencies( + requires_dist, # type: Optional[List[Dependency]] + parent=None, # type: Optional[Dependency] +): + # type: (...) -> Optional[Set[Dependency]] + if requires_dist is None: + return None + dependencies = set() + for req in requires_dist: + if not isinstance(req, Dependency): + dependencies.add(Dependency.from_str(req, parent=parent)) + else: + dependencies.add(req) + return dependencies + + +@attr.s(frozen=True) +class PackageInfo(object): + name = attr.ib(type=str) + version = attr.ib(type=str) + package_url = attr.ib(type=str) + summary = attr.ib(type=str, default=None) # type: Optional[str] + author = attr.ib(type=str, default=None) # type: Optional[str] + keywords = attr.ib(factory=list, converter=split_keywords) # type: List[str] + description = attr.ib(type=str, default="") + download_url = attr.ib(type=str, default="") + home_page = attr.ib(type=str, default="") + license = attr.ib(type=str, default="") + maintainer = attr.ib(type=str, default="") + maintainer_email = attr.ib(type=str, default="") + downloads = attr.ib(factory=dict) # type: Dict[str, int] + docs_url = attr.ib(default=None) # type: Optional[str] + platform = attr.ib(type=str, default="") + project_url = attr.ib(type=str, default="") + project_urls = attr.ib(factory=dict) # type: Dict[str, str] + requires_python = attr.ib(default=None) # type: Optional[str] + requires_dist = attr.ib(factory=list) # type: List[Dependency] + release_url = attr.ib(default=None) # type: Optional[str] + description_content_type = attr.ib(type=str, default="text/md") + bugtrack_url = attr.ib(default=None) # type: str + classifiers = attr.ib(factory=list) # type: List[str] + author_email = attr.ib(default=None) # type: Optional[str] + markers = attr.ib(default=None) # type: Optional[str] + dependencies = attr.ib(default=None) # type: Tuple[Dependency] + + @classmethod + def from_json(cls, info_json): + # type: (TPackageInfo) -> "PackageInfo" + return cls(**filter_dict(info_json)) # type: ignore + + def to_dependency(self): + # type: () -> Dependency + return Dependency.from_info(self) + + def create_dependencies(self, force=False): + # type: (bool) -> "PackageInfo" + """ + Create values for **self.dependencies**. + + :param bool force: Sets **self.dependencies** to an empty tuple if it would be + None, defaults to False. + :return: An updated instance of the current object with **self.dependencies** + updated accordingly. + :rtype: :class:`PackageInfo` + """ + if not self.dependencies and not self.requires_dist: + if force: + return attr.evolve(self, dependencies=tuple()) + return self + self_dependency = self.to_dependency() + deps = set() + self_dependencies = tuple() if not self.dependencies else self.dependencies + for dep in self_dependencies: + if dep is None: + continue + new_dep = dep.add_parent(self_dependency) + deps.add(new_dep) + created_deps = create_dependencies(self.requires_dist, parent=self_dependency) + if created_deps is not None: + for dep in created_deps: + if dep is None: + continue + deps.add(dep) + return attr.evolve(self, dependencies=tuple(sorted(deps))) + + +def convert_package_info(info_json): + # type: (Union[TPackageInfo, PackageInfo]) -> PackageInfo + if isinstance(info_json, PackageInfo): + return info_json + return PackageInfo.from_json(info_json) + + +def add_markers_to_dep(d, marker_str): + # type: (str, Union[str, Marker]) -> str + req = PackagingRequirement(d) + existing_marker = getattr(req, "marker", None) + if isinstance(marker_str, Marker): + marker_str = str(marker_str) + if existing_marker is not None: + marker_str = str(merge_markers(existing_marker, marker_str)) + if marker_str: + marker_str = marker_str.replace("'", '"') + req.marker = Marker(marker_str) + return str(req) + + +@attr.s +class Package(object): + info = attr.ib(type=PackageInfo, converter=convert_package_info) + last_serial = attr.ib(type=int) + releases = attr.ib( + type=ReleaseCollection, + converter=instance_check_converter( # type: ignore + ReleaseCollection, convert_releases_to_collection + ), + ) + # XXX: Note: sometimes releases have no urls at the top level (e.g. pyrouge) + urls = attr.ib( + type=ReleaseUrlCollection, + converter=instance_check_converter( # type: ignore + ReleaseUrlCollection, convert_release_urls_to_collection + ), + ) + + @urls.default + def _get_urls_collection(self): + return functools.partial(convert_release_urls_to_collection, urls=[], name=self.name) + + @property + def name(self): + # type: () -> str + return self.info.name + + @property + def version(self): + # type: () -> str + return self.info.version + + @property + def requirement(self): + # type: () -> PackagingRequirement + return self.info.to_dependency().requirement + + @property + def latest_sdist(self): + # type: () -> ReleaseUrl + return next(iter(self.urls.sdists)) + + @property + def latest_wheels(self): + # type: () -> Iterator[ReleaseUrl] + for wheel in self.urls.wheels: + yield wheel + + @property + def dependencies(self): + # type: () -> List[Dependency] + if self.info.dependencies is None and list(self.urls): + rval = self.get_dependencies() + return rval.dependencies + return list(self.info.dependencies) + + def get_dependencies(self): + # type: () -> "Package" + urls = [] # type: List[ReleaseUrl] + deps = set() # type: Set[str] + info = self.info + if info.dependencies is None: + for url in self.urls: + try: + url, dep_dict = url.get_dependencies() + except (RuntimeError, TypeError): + # This happens if we are parsing `setup.py` and we fail + if url.is_sdist: + continue + else: + raise + markers = url.markers + dep_list = dep_dict.get("requires_dist", []) + for dep in dep_list: + # XXX: We need to parse these as requirements and "and" the markers + # XXX: together because they may contain "extra" markers which we + # XXX: will need to parse and remove + deps.add(add_markers_to_dep(dep, markers)) + urls.append(url) + if None in deps: + deps.remove(None) + info = attr.evolve( + self.info, requires_dist=tuple(sorted(deps)) + ).create_dependencies(force=True) + return attr.evolve(self, info=info, urls=urls) + + @classmethod + def from_json(cls, package_json): + # type: (Dict[str, Any]) -> "Package" + info = convert_package_info(package_json["info"]).create_dependencies() + releases = convert_releases_to_collection( + package_json["releases"], name=info.name + ) + urls = convert_release_urls_to_collection(package_json["urls"], name=info.name) + return cls( + info=info, + releases=releases, + urls=urls, + last_serial=package_json["last_serial"], + ) + + def pin_dependencies(self, include_extras=None): + # type: (Optional[List[str]]) -> Tuple[List["Package"], Dict[str, List[SpecifierSet]]] + deps = [] + if include_extras: + include_extras = list(sorted(set(include_extras))) + else: + include_extras = [] + constraints = defaultdict(list) + for dep in self.dependencies: + if dep.from_extras and dep.from_extras not in include_extras: + continue + if dep.specifier: + constraints[dep.name].append(dep.specifier) + try: + pinned = dep.pin() + except requests.exceptions.HTTPError: + continue + deps.append(pinned) + return deps, constraints + + def get_latest_lockfile(self): + # type: () -> Dict[str, Dict[str, Union[List[str], str]]] + lockfile = {} + constraints = {dep.name: dep.specifier for dep in self.dependencies} + deps, _ = self.pin_dependencies() + for dep in deps: + dep = dep.get_dependencies() + for sub_dep in dep.dependencies: + if sub_dep.name not in constraints: + logger.info( + "Adding {0} (from {1}) {2!s}".format( + sub_dep.name, dep.name, sub_dep.specifier + ) + ) + constraints[sub_dep.name] = sub_dep.specifier + else: + existing = "{0} (from {1}): {2!s} + ".format( + sub_dep.name, dep.name, constraints[sub_dep.name] + ) + new_specifier = sub_dep.specifier + merged = constraints[sub_dep.name] & new_specifier + logger.info( + "Updating: {0}{1!s} = {2!s}".format( + existing, new_specifier, merged + ) + ) + constraints[sub_dep.name] = merged + + lockfile.update({dep.info.name: dep.releases.get_latest_lockfile()}) + for sub_dep_name, specset in constraints.items(): + try: + sub_dep_pkg = get_package(sub_dep_name) + except requests.exceptions.HTTPError: + continue + logger.info("Getting package: {0} ({1!s})".format(sub_dep, specset)) + sorted_releases = list( + sorted( + sub_dep_pkg.releases, + key=operator.attrgetter("parsed_version"), + reverse=True, + ) + ) + try: + version = next(iter(specset.filter((r.version for r in sorted_releases)))) + except StopIteration: + logger.info("No version of {0} matches specifier: {1}".format(sub_dep, specset)) + logger.info( + "Available versions: {0}".format( + " ".join([r.version for r in sorted_releases]) + ) + ) + raise + sub_dep_instance = get_package_version(sub_dep_name, version=str(version)) + if sub_dep_instance is None: + continue + lockfile.update( + { + sub_dep_instance.info.name: sub_dep_instance.releases.get_latest_lockfile() + } + ) + # lockfile.update(dep.get_latest_lockfile()) + lockfile.update({self.info.name: self.releases.get_latest_lockfile()}) + return lockfile + + def as_dict(self): + # type: () -> Dict[str, Any] + return json.loads(self.serialize()) + + def serialize(self): + # type: () -> str + return json.dumps(attr.asdict(self), cls=PackageEncoder, indent=4) + + +def get_package(name): + # type: (str) -> Package + url = "https://pypi.org/pypi/{}/json".format(name) + with requests.get(url) as r: + r.raise_for_status() + result = r.json() + package = Package.from_json(result) + return package + + +def get_package_version(name, version): + # type: (str, str) -> Package + url = "https://pypi.org/pypi/{0}/{1}/json".format(name, version) + with requests.get(url) as r: + r.raise_for_status() + result = r.json() + package = Package.from_json(result) + return package + + +def get_package_from_requirement(req): + # type: (PackagingRequirement) -> Tuple[Package, Set[str]] + versions = set() + if is_pinned_requirement(req): + version = get_pinned_version(req) + versions.add(version) + pkg = get_package_version(req.name, version) + else: + pkg = get_package(req.name) + sorted_releases = list( + sorted(pkg.releases, key=operator.attrgetter("parsed_version"), reverse=True) + ) + versions = set(req.specifier.filter((r.version for r in sorted_releases))) + version = next(iter(req.specifier.filter((r.version for r in sorted_releases)))) + if pkg.version not in versions: + pkg = get_package_version(pkg.name, version) + return pkg, versions diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index db1f5dde..9c0aea4e 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -164,6 +164,10 @@ class Pipfile(object): # type: () -> Union[plette.pipfiles.Pipfile, PipfileLoader] return self.projectfile.model + @property + def root(self): + return self.path.parent + @property def extended_keys(self): return [ diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 81b4b715..610eb68b 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -679,7 +679,7 @@ AST_COMPARATORS = dict( (ast.And, operator.and_), (ast.Or, operator.or_), (ast.Not, operator.not_), - (ast.In, operator.contains), + (ast.In, lambda a, b: operator.contains(b, a)), ) ) @@ -688,6 +688,8 @@ class Analyzer(ast.NodeVisitor): def __init__(self): self.name_types = [] self.function_map = {} # type: Dict[Any, Any] + self.function_names = {} + self.resolved_function_names = {} self.functions = [] self.strings = [] self.assignments = {} @@ -725,6 +727,38 @@ class Analyzer(ast.NodeVisitor): iter(k for k in self.assignments if getattr(k, "id", "") == match.id), None ) + def parse_function_names(self, should_retry=True, function_map=None): + if function_map is None: + function_map = {} + retries = [] + for k, v in function_map.items(): + fn_name = "" + if k in self.function_names: + fn_name = self.function_names[k] + elif isinstance(k, ast.Name): + fn_name = k.id + elif isinstance(k, ast.Attribute): + try: + fn = ast_unparse(k, analyzer=self) + except Exception: + if should_retry: + retries.append((k, v)) + continue + else: + if isinstance(fn, six.string_types): + _, _, fn_name = fn.rpartition(".") + if fn_name: + self.resolved_function_names[fn_name] = ast_unparse(v, analyzer=self) + return retries + + def parse_functions(self): + retries = self.parse_function_names(function_map=self.function_map) + if retries: + failures = self.parse_function_names( + should_retry=False, function_map=dict(retries) + ) + return self.resolved_function_names + def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # noqa:C901 # type: (Any, bool, Optional[Analyzer], bool) -> Union[List[Any], Dict[Any, Any], Tuple[Any, ...], STRING_TYPE] @@ -746,6 +780,13 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no unparsed = item.s elif isinstance(item, ast.Subscript): unparsed = unparse(item.value) + if not initial_mapping: + if isinstance(item.slice, ast.Index): + try: + unparsed = unparsed[unparse(item.slice.value)] + except KeyError: + # not everything can be looked up before runtime + unparsed = item elif any(isinstance(item, k) for k in AST_BINOP_MAP.keys()): unparsed = AST_BINOP_MAP[type(item)] elif isinstance(item, ast.Num): @@ -789,7 +830,7 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no elif isinstance(item, constant): unparsed = item.value elif isinstance(item, ast.Compare): - if isinstance(item.left, ast.Attribute): + if isinstance(item.left, ast.Attribute) or isinstance(item.left, ast.Str): import importlib left = unparse(item.left) @@ -934,18 +975,10 @@ def ast_parse_setup_py(path): ast_analyzer = ast_parse_file(path) setup = {} # type: Dict[Any, Any] ast_analyzer.unmap_binops() - for k, v in ast_analyzer.function_map.items(): - fn_name = "" - if isinstance(k, ast.Name): - fn_name = k.id - elif isinstance(k, ast.Attribute): - fn = ast_unparse(k) - if isinstance(fn, six.string_types): - _, _, fn_name = fn.rpartition(".") - if fn_name == "setup": - setup = v - cleaned_setup = ast_unparse(setup, analyzer=ast_analyzer) - return cleaned_setup + function_names = ast_analyzer.parse_functions() + if "setup" in function_names: + setup = ast_unparse(function_names["setup"], analyzer=ast_analyzer) + return setup def run_setup(script_path, egg_base=None): @@ -1557,7 +1590,7 @@ build-backend = "{1}" from .dependencies import get_finder session, finder = get_finder() - _, uri = split_vcs_method_from_uri(unquote(ireq.link.url_without_fragment)) + vcs, uri = split_vcs_method_from_uri(unquote(ireq.link.url_without_fragment)) parsed = urlparse(uri) if "file" in parsed.scheme: url_path = parsed.path @@ -1571,14 +1604,10 @@ build-backend = "{1}" uri = uri.replace("file:/", "file:///") path = pip_shims.shims.url_to_path(uri) kwargs = _prepare_wheel_building_kwargs(ireq) - ireq.source_dir = kwargs["src_dir"] - try: - is_vcs = ireq.link.is_vcs - except AttributeError: - try: - is_vcs = not ireq.link.is_artifact - except AttributeError: - is_vcs = False + is_artifact_or_vcs = getattr( + ireq.link, "is_vcs", getattr(ireq.link, "is_artifact", False) + ) + is_vcs = True if vcs else is_artifact_or_vcs if not (ireq.editable and pip_shims.shims.is_file_url(ireq.link) and is_vcs): if ireq.is_wheel: only_download = True @@ -1599,7 +1628,6 @@ build-backend = "{1}" ireq.ensure_has_source_dir(kwargs["src_dir"]) src_dir = ireq.source_dir with pip_shims.shims.global_tempdir_manager(): - ireq.populate_link(finder, False, False) pip_shims.shims.shim_unpack( link=ireq.link, location=kwargs["src_dir"], @@ -1609,7 +1637,9 @@ build-backend = "{1}" hashes=ireq.hashes(False), progress_bar="off", ) - created = cls.create(src_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs) + created = cls.create( + kwargs["src_dir"], subdirectory=subdir, ireq=ireq, kwargs=kwargs + ) return created @classmethod diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 9b6beb64..6c3b7de8 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -723,7 +723,7 @@ def get_pinned_version(ireq): except AttributeError: raise TypeError("Expected InstallRequirement, not {}".format(type(ireq).__name__)) - if ireq.editable: + if getattr(ireq, "editable", False): raise ValueError("InstallRequirement is editable") if not specifier: raise ValueError("InstallRequirement has no version specification") diff --git a/pipenv/vendor/resolvelib/__init__.py b/pipenv/vendor/resolvelib/__init__.py index e0e37434..aaba5b3a 100644 --- a/pipenv/vendor/resolvelib/__init__.py +++ b/pipenv/vendor/resolvelib/__init__.py @@ -1,16 +1,26 @@ __all__ = [ - '__version__', - 'AbstractProvider', 'BaseReporter', 'Resolver', - 'NoVersionsAvailable', 'RequirementsConflicted', - 'ResolutionError', 'ResolutionImpossible', 'ResolutionTooDeep', + "__version__", + "AbstractProvider", + "AbstractResolver", + "BaseReporter", + "InconsistentCandidate", + "Resolver", + "RequirementsConflicted", + "ResolutionError", + "ResolutionImpossible", + "ResolutionTooDeep", ] -__version__ = '0.2.2' +__version__ = "0.3.0" -from .providers import AbstractProvider +from .providers import AbstractProvider, AbstractResolver from .reporters import BaseReporter from .resolvers import ( - NoVersionsAvailable, RequirementsConflicted, - Resolver, ResolutionError, ResolutionImpossible, ResolutionTooDeep, + InconsistentCandidate, + RequirementsConflicted, + Resolver, + ResolutionError, + ResolutionImpossible, + ResolutionTooDeep, ) diff --git a/pipenv/vendor/resolvelib/providers.py b/pipenv/vendor/resolvelib/providers.py index 515c0db4..db168219 100644 --- a/pipenv/vendor/resolvelib/providers.py +++ b/pipenv/vendor/resolvelib/providers.py @@ -1,6 +1,7 @@ class AbstractProvider(object): - """Delegate class to provide requirment interface for the resolver. + """Delegate class to provide requirement interface for the resolver. """ + def identify(self, dependency): """Given a dependency, return an identifier for it. @@ -56,15 +57,15 @@ class AbstractProvider(object): consulted to find concrete candidates for this requirement. The returned candidates should be sorted by reversed preference, e.g. - the latest should be LAST. This is done so list-popping can be as - efficient as possible. + the most preferred should be LAST. This is done so list-popping can be + as efficient as possible. """ raise NotImplementedError def is_satisfied_by(self, requirement, candidate): """Whether the given requirement can be satisfied by a candidate. - A boolean should be retuened to indicate whether `candidate` is a + A boolean should be returned to indicate whether `candidate` is a viable solution to the requirement. """ raise NotImplementedError @@ -76,3 +77,45 @@ class AbstractProvider(object): specifies as its dependencies. """ raise NotImplementedError + + +class AbstractResolver(object): + """The thing that performs the actual resolution work. + """ + + base_exception = Exception + + def __init__(self, provider, reporter): + self.provider = provider + self.reporter = reporter + + def resolve(self, requirements, **kwargs): + """Take a collection of constraints, spit out the resolution result. + + Parameters + ---------- + requirements : Collection + A collection of constraints + kwargs : optional + Additional keyword arguments that subclasses may accept. + + Raises + ------ + self.base_exception + Any raised exception is guaranteed to be a subclass of + self.base_exception. The string representation of an exception + should be human readable and provide context for why it occurred. + + Returns + ------- + retval : object + A representation of the final resolution state. It can be any object + with a `mapping` attribute that is a Mapping. Other attributes can + be used to provide resolver-specific information. + + The `mapping` attribute MUST be key-value pair is an identifier of a + requirement (as returned by the provider's `identify` method) mapped + to the resolved candidate (chosen from the return value of the + provider's `find_matches` method). + """ + raise NotImplementedError diff --git a/pipenv/vendor/resolvelib/reporters.py b/pipenv/vendor/resolvelib/reporters.py index c723031f..c7e9e88b 100644 --- a/pipenv/vendor/resolvelib/reporters.py +++ b/pipenv/vendor/resolvelib/reporters.py @@ -1,6 +1,7 @@ class BaseReporter(object): """Delegate class to provider progress reporting for the resolver. """ + def starting(self): """Called before the resolution actually starts. """ @@ -21,3 +22,15 @@ class BaseReporter(object): def ending(self, state): """Called before the resolution ends successfully. """ + + def adding_requirement(self, requirement): + """Called when the resolver adds a new requirement into the resolve criteria. + """ + + def backtracking(self, candidate): + """Called when the resolver rejects a candidate during backtracking. + """ + + def pinning(self, candidate): + """Called when adding a candidate to the potential solution. + """ diff --git a/pipenv/vendor/resolvelib/resolvers.py b/pipenv/vendor/resolvelib/resolvers.py index 9c69628e..b51d337d 100644 --- a/pipenv/vendor/resolvelib/resolvers.py +++ b/pipenv/vendor/resolvelib/resolvers.py @@ -1,52 +1,91 @@ import collections +from .providers import AbstractResolver from .structs import DirectedGraph -RequirementInformation = collections.namedtuple('RequirementInformation', [ - 'requirement', 'parent', -]) +RequirementInformation = collections.namedtuple( + "RequirementInformation", ["requirement", "parent"] +) -class NoVersionsAvailable(Exception): - def __init__(self, requirement, parent): - super(NoVersionsAvailable, self).__init__() - self.requirement = requirement - self.parent = parent +class ResolverException(Exception): + """A base class for all exceptions raised by this module. + + Exceptions derived by this class should all be handled in this module. Any + bubbling pass the resolver should be treated as a bug. + """ -class RequirementsConflicted(Exception): +class RequirementsConflicted(ResolverException): def __init__(self, criterion): - super(RequirementsConflicted, self).__init__() + super(RequirementsConflicted, self).__init__(criterion) self.criterion = criterion + def __str__(self): + return "Requirements conflict: {}".format( + ", ".join(repr(r) for r in self.criterion.iter_requirement()), + ) + + +class InconsistentCandidate(ResolverException): + def __init__(self, candidate, criterion): + super(InconsistentCandidate, self).__init__(candidate, criterion) + self.candidate = candidate + self.criterion = criterion + + def __str__(self): + return "Provided candidate {!r} does not satisfy {}".format( + self.candidate, + ", ".join(repr(r) for r in self.criterion.iter_requirement()), + ) + class Criterion(object): - """Internal representation of possible resolution results of a package. + """Representation of possible resolution results of a package. - This holds two attributes: + This holds three attributes: - * `information` is a collection of `RequirementInformation` pairs. Each - pair is a requirement contributing to this criterion, and the candidate - that provides the requirement. + * `information` is a collection of `RequirementInformation` pairs. + Each pair is a requirement contributing to this criterion, and the + candidate that provides the requirement. + * `incompatibilities` is a collection of all known not-to-work candidates + to exclude from consideration. * `candidates` is a collection containing all possible candidates deducted - from the union of contributing requirements. It should never be empty. + from the union of contributing requirements and known incompatibilities. + It should never be empty, except when the criterion is an attribute of a + raised `RequirementsConflicted` (in which case it is always empty). + + .. note:: + This class is intended to be externally immutable. **Do not** mutate + any of its attribute containers. """ - def __init__(self, candidates, information): + + def __init__(self, candidates, information, incompatibilities): self.candidates = candidates self.information = information + self.incompatibilities = incompatibilities + + def __repr__(self): + requirements = ", ".join( + "{!r} from {!r}".format(req, parent) + for req, parent in self.information + ) + return "".format(requirements) @classmethod def from_requirement(cls, provider, requirement, parent): """Build an instance from a requirement. """ candidates = provider.find_matches(requirement) - if not candidates: - raise NoVersionsAvailable(requirement, parent) - return cls( + criterion = cls( candidates=candidates, information=[RequirementInformation(requirement, parent)], + incompatibilities=[], ) + if not candidates: + raise RequirementsConflicted(criterion) + return criterion def iter_requirement(self): return (i.requirement for i in self.information) @@ -60,22 +99,38 @@ class Criterion(object): infos = list(self.information) infos.append(RequirementInformation(requirement, parent)) candidates = [ - c for c in self.candidates + c + for c in self.candidates if provider.is_satisfied_by(requirement, c) ] + criterion = type(self)(candidates, infos, list(self.incompatibilities)) if not candidates: - raise RequirementsConflicted(self) - return type(self)(candidates, infos) + raise RequirementsConflicted(criterion) + return criterion + + def excluded_of(self, candidate): + """Build a new instance from this, but excluding specified candidate. + + Returns the new instance, or None if we still have no valid candidates. + """ + incompats = list(self.incompatibilities) + incompats.append(candidate) + candidates = [c for c in self.candidates if c != candidate] + if not candidates: + return None + criterion = type(self)(candidates, list(self.information), incompats) + return criterion -class ResolutionError(Exception): +class ResolutionError(ResolverException): pass class ResolutionImpossible(ResolutionError): - def __init__(self, requirements): - super(ResolutionImpossible, self).__init__() - self.requirements = requirements + def __init__(self, causes): + super(ResolutionImpossible, self).__init__(causes) + # causes is a list of RequirementInformation objects + self.causes = causes class ResolutionTooDeep(ResolutionError): @@ -85,7 +140,7 @@ class ResolutionTooDeep(ResolutionError): # Resolution state in a round. -State = collections.namedtuple('State', 'mapping graph') +State = collections.namedtuple("State", "mapping criteria") class Resolution(object): @@ -94,10 +149,10 @@ class Resolution(object): This is designed as a one-off object that holds information to kick start the resolution process, and holds the results afterwards. """ + def __init__(self, provider, reporter): self._p = provider self._r = reporter - self._criteria = {} self._states = [] @property @@ -105,7 +160,7 @@ class Resolution(object): try: return self._states[-1] except IndexError: - raise AttributeError('state') + raise AttributeError("state") def _push_new_state(self): """Push a new state into history. @@ -116,30 +171,29 @@ class Resolution(object): try: base = self._states[-1] except IndexError: - graph = DirectedGraph() - graph.add(None) # Sentinel as root dependencies' parent. - state = State(mapping={}, graph=graph) + state = State(mapping=collections.OrderedDict(), criteria={}) else: state = State( - mapping=base.mapping.copy(), - graph=base.graph.copy(), + mapping=base.mapping.copy(), criteria=base.criteria.copy(), ) self._states.append(state) - def _contribute_to_criteria(self, name, requirement, parent): + def _merge_into_criterion(self, requirement, parent): + self._r.adding_requirement(requirement) + name = self._p.identify(requirement) try: - crit = self._criteria[name] + crit = self.state.criteria[name] except KeyError: crit = Criterion.from_requirement(self._p, requirement, parent) else: crit = crit.merged_with(self._p, requirement, parent) - self._criteria[name] = crit + return name, crit def _get_criterion_item_preference(self, item): name, criterion = item try: pinned = self.state.mapping[name] - except (IndexError, KeyError): + except KeyError: pinned = None return self._p.get_preference( pinned, criterion.candidates, criterion.information, @@ -155,114 +209,183 @@ class Resolution(object): for r in criterion.iter_requirement() ) - def _check_pinnability(self, candidate, dependencies): - backup = self._criteria.copy() - contributed = set() - try: - for subdep in dependencies: - key = self._p.identify(subdep) - self._contribute_to_criteria(key, subdep, parent=candidate) - contributed.add(key) - except RequirementsConflicted: - self._criteria = backup - return None - return contributed + def _get_criteria_to_update(self, candidate): + criteria = {} + for r in self._p.get_dependencies(candidate): + name, crit = self._merge_into_criterion(r, parent=candidate) + criteria[name] = crit + return criteria - def _pin_candidate(self, name, criterion, candidate, child_names): - try: - self.state.graph.remove(name) - except KeyError: - pass - self.state.mapping[name] = candidate - self.state.graph.add(name) - for parent in criterion.iter_parent(): - parent_name = None if parent is None else self._p.identify(parent) + def _attempt_to_pin_criterion(self, name, criterion): + causes = [] + for candidate in reversed(criterion.candidates): try: - self.state.graph.connect(parent_name, name) - except KeyError: - # Parent is not yet pinned. Skip now; this edge will be - # connected when the parent is being pinned. - pass - for child_name in child_names: - try: - self.state.graph.connect(name, child_name) - except KeyError: - # Child is not yet pinned. Skip now; this edge will be - # connected when the child is being pinned. - pass - - def _pin_criteria(self): - criterion_names = [name for name, _ in sorted( - self._criteria.items(), - key=self._get_criterion_item_preference, - )] - for name in criterion_names: - # Any pin may modify any criterion during the loop. Criteria are - # replaced, not updated in-place, so we need to read this value - # in the loop instead of outside. (sarugaku/resolvelib#5) - criterion = self._criteria[name] - - if self._is_current_pin_satisfying(name, criterion): - # If the current pin already works, just use it. + criteria = self._get_criteria_to_update(candidate) + except RequirementsConflicted as e: + causes.append(e.criterion) continue - candidates = list(criterion.candidates) - while candidates: - candidate = candidates.pop() - dependencies = self._p.get_dependencies(candidate) - child_names = self._check_pinnability(candidate, dependencies) - if child_names is None: - continue - self._pin_candidate(name, criterion, candidate, child_names) - break - else: # All candidates tried, nothing works. Give up. (?) - raise ResolutionImpossible(list(criterion.iter_requirement())) + + # Put newly-pinned candidate at the end. This is essential because + # backtracking looks at this mapping to get the last pin. + self._r.pinning(candidate) + self.state.mapping.pop(name, None) + self.state.mapping[name] = candidate + self.state.criteria.update(criteria) + + # Check the newly-pinned candidate actually works. This should + # always pass under normal circumstances, but in the case of a + # faulty provider, we will raise an error to notify the implementer + # to fix find_matches() and/or is_satisfied_by(). + if not self._is_current_pin_satisfying(name, criterion): + raise InconsistentCandidate(candidate, criterion) + + return [] + + # All candidates tried, nothing works. This criterion is a dead + # end, signal for backtracking. + return causes + + def _backtrack(self): + # We need at least 3 states here: + # (a) One known not working, to drop. + # (b) One to backtrack to. + # (c) One to restore state (b) to its state prior to candidate-pinning, + # so we can pin another one instead. + while len(self._states) >= 3: + del self._states[-1] + + # Retract the last candidate pin, and create a new (b). + name, candidate = self._states.pop().mapping.popitem() + self._r.backtracking(candidate) + self._push_new_state() + + # Mark the retracted candidate as incompatible. + criterion = self.state.criteria[name].excluded_of(candidate) + if criterion is None: + # This state still does not work. Try the still previous state. + continue + self.state.criteria[name] = criterion + + return True + + return False def resolve(self, requirements, max_rounds): if self._states: - raise RuntimeError('already resolved') + raise RuntimeError("already resolved") - for requirement in requirements: + self._push_new_state() + for r in requirements: try: - name = self._p.identify(requirement) - self._contribute_to_criteria(name, requirement, parent=None) + name, crit = self._merge_into_criterion(r, parent=None) except RequirementsConflicted as e: - # If initial requirements conflict, nothing would ever work. - raise ResolutionImpossible(e.requirements + [requirement]) + raise ResolutionImpossible(e.criterion.information) + self.state.criteria[name] = crit - last = None self._r.starting() for round_index in range(max_rounds): self._r.starting_round(round_index) self._push_new_state() - self._pin_criteria() - curr = self.state - if last is not None and len(curr.mapping) == len(last.mapping): - # Nothing new added. Done! Remove the duplicated entry. + + unsatisfied_criterion_items = [ + item + for item in self.state.criteria.items() + if not self._is_current_pin_satisfying(*item) + ] + + # All criteria are accounted for. Nothing more to pin, we are done! + if not unsatisfied_criterion_items: del self._states[-1] - self._r.ending(last) - return - last = curr + self._r.ending(curr) + return self.state + + # Choose the most preferred unpinned criterion to try. + name, criterion = min( + unsatisfied_criterion_items, + key=self._get_criterion_item_preference, + ) + failure_causes = self._attempt_to_pin_criterion(name, criterion) + + # Backtrack if pinning fails. + if failure_causes: + result = self._backtrack() + if not result: + causes = [ + i for crit in failure_causes for i in crit.information + ] + raise ResolutionImpossible(causes) self._r.ending_round(round_index, curr) raise ResolutionTooDeep(max_rounds) -class Resolver(object): +def _has_route_to_root(criteria, key, all_keys, connected): + if key in connected: + return True + if key not in criteria: + return False + for p in criteria[key].iter_parent(): + try: + pkey = all_keys[id(p)] + except KeyError: + continue + if pkey in connected: + connected.add(key) + return True + if _has_route_to_root(criteria, pkey, all_keys, connected): + connected.add(key) + return True + return False + + +Result = collections.namedtuple("Result", "mapping graph criteria") + + +def _build_result(state): + mapping = state.mapping + all_keys = {id(v): k for k, v in mapping.items()} + all_keys[id(None)] = None + + graph = DirectedGraph() + graph.add(None) # Sentinel as root dependencies' parent. + + connected = {None} + for key, criterion in state.criteria.items(): + if not _has_route_to_root(state.criteria, key, all_keys, connected): + continue + if key not in graph: + graph.add(key) + for p in criterion.iter_parent(): + try: + pkey = all_keys[id(p)] + except KeyError: + continue + if pkey not in graph: + graph.add(pkey) + graph.connect(pkey, key) + + return Result( + mapping={k: v for k, v in mapping.items() if k in connected}, + graph=graph, + criteria=state.criteria, + ) + + +class Resolver(AbstractResolver): """The thing that performs the actual resolution work. """ - def __init__(self, provider, reporter): - self.provider = provider - self.reporter = reporter - def resolve(self, requirements, max_rounds=20): + base_exception = ResolverException + + def resolve(self, requirements, max_rounds=100): """Take a collection of constraints, spit out the resolution result. The return value is a representation to the final resolution result. It - is a tuple subclass with two public members: + is a tuple subclass with three public members: * `mapping`: A dict of resolved candidates. Each key is an identifier of a requirement (as returned by the provider's `identify` method), @@ -271,17 +394,21 @@ class Resolver(object): The vertices are keys of `mapping`, and each edge represents *why* a particular package is included. A special vertex `None` is included to represent parents of user-supplied requirements. + * `criteria`: A dict of "criteria" that hold detailed information on + how edges in the graph are derived. Each key is an identifier of a + requirement, and the value is a `Criterion` instance. The following exceptions may be raised if a resolution cannot be found: - * `NoVersionsAvailable`: A requirement has no available candidates. * `ResolutionImpossible`: A resolution cannot be found for the given - combination of requirements. + combination of requirements. The `causes` attribute of the + exception is a list of (requirement, parent), giving the + requirements that could not be satisfied. * `ResolutionTooDeep`: The dependency tree is too deeply nested and the resolver gave up. This is usually caused by a circular dependency, but you can try to resolve this by increasing the `max_rounds` argument. """ resolution = Resolution(self.provider, self.reporter) - resolution.resolve(requirements, max_rounds=max_rounds) - return resolution.state + state = resolution.resolve(requirements, max_rounds=max_rounds) + return _build_result(state) diff --git a/pipenv/vendor/resolvelib/structs.py b/pipenv/vendor/resolvelib/structs.py index 97bd0095..1eee08b3 100644 --- a/pipenv/vendor/resolvelib/structs.py +++ b/pipenv/vendor/resolvelib/structs.py @@ -1,10 +1,11 @@ class DirectedGraph(object): """A graph structure with directed edges. """ + def __init__(self): self._vertices = set() - self._forwards = {} # -> Set[] - self._backwards = {} # -> Set[] + self._forwards = {} # -> Set[] + self._backwards = {} # -> Set[] def __iter__(self): return iter(self._vertices) @@ -28,7 +29,7 @@ class DirectedGraph(object): """Add a new vertex to the graph. """ if key in self._vertices: - raise ValueError('vertex exists') + raise ValueError("vertex exists") self._vertices.add(key) self._forwards[key] = set() self._backwards[key] = set() diff --git a/pipenv/vendor/urllib3/__init__.py b/pipenv/vendor/urllib3/__init__.py index 9bd8323f..667e9bce 100644 --- a/pipenv/vendor/urllib3/__init__.py +++ b/pipenv/vendor/urllib3/__init__.py @@ -22,7 +22,7 @@ from logging import NullHandler __author__ = "Andrey Petrov (andrey.petrov@shazow.net)" __license__ = "MIT" -__version__ = "1.25.8" +__version__ = "1.25.9" __all__ = ( "HTTPConnectionPool", diff --git a/pipenv/vendor/urllib3/connection.py b/pipenv/vendor/urllib3/connection.py index 71e6790b..6da1cf4b 100644 --- a/pipenv/vendor/urllib3/connection.py +++ b/pipenv/vendor/urllib3/connection.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +import re import datetime import logging import os @@ -58,6 +59,8 @@ port_by_scheme = {"http": 80, "https": 443} # (ie test_recent_date is failing) update it to ~6 months before the current date. RECENT_DATE = datetime.date(2019, 1, 1) +_CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") + class DummyConnection(object): """Used to detect a failed ConnectionCls import.""" @@ -184,6 +187,17 @@ class HTTPConnection(_HTTPConnection, object): conn = self._new_conn() self._prepare_conn(conn) + def putrequest(self, method, url, *args, **kwargs): + """Send a request to the server""" + match = _CONTAINS_CONTROL_CHAR_RE.search(method) + if match: + raise ValueError( + "Method cannot contain non-token characters %r (found at least %r)" + % (method, match.group()) + ) + + return _HTTPConnection.putrequest(self, method, url, *args, **kwargs) + def request_chunked(self, method, url, body=None, headers=None): """ Alternative to the common request method, which sends the @@ -223,7 +237,12 @@ class HTTPConnection(_HTTPConnection, object): class HTTPSConnection(HTTPConnection): default_port = port_by_scheme["https"] + cert_reqs = None + ca_certs = None + ca_cert_dir = None + ca_cert_data = None ssl_version = None + assert_fingerprint = None def __init__( self, @@ -251,19 +270,6 @@ class HTTPSConnection(HTTPConnection): # HTTPS requests to go out as HTTP. (See Issue #356) self._protocol = "https" - -class VerifiedHTTPSConnection(HTTPSConnection): - """ - Based on httplib.HTTPSConnection but wraps the socket with - SSL certification. - """ - - cert_reqs = None - ca_certs = None - ca_cert_dir = None - ssl_version = None - assert_fingerprint = None - def set_cert( self, key_file=None, @@ -274,6 +280,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): assert_hostname=None, assert_fingerprint=None, ca_cert_dir=None, + ca_cert_data=None, ): """ This method should only be called once, before the connection is used. @@ -294,6 +301,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): self.assert_fingerprint = assert_fingerprint self.ca_certs = ca_certs and os.path.expanduser(ca_certs) self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) + self.ca_cert_data = ca_cert_data def connect(self): # Add certificate verification @@ -344,6 +352,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): if ( not self.ca_certs and not self.ca_cert_dir + and not self.ca_cert_data and default_ssl_context and hasattr(context, "load_default_certs") ): @@ -356,6 +365,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): key_password=self.key_password, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, + ca_cert_data=self.ca_cert_data, server_hostname=server_hostname, ssl_context=context, ) @@ -406,9 +416,8 @@ def _match_hostname(cert, asserted_hostname): raise -if ssl: - # Make a copy for testing. - UnverifiedHTTPSConnection = HTTPSConnection - HTTPSConnection = VerifiedHTTPSConnection -else: - HTTPSConnection = DummyConnection +if not ssl: + HTTPSConnection = DummyConnection # noqa: F811 + + +VerifiedHTTPSConnection = HTTPSConnection diff --git a/pipenv/vendor/urllib3/connectionpool.py b/pipenv/vendor/urllib3/connectionpool.py index d42eb7be..5f044dbd 100644 --- a/pipenv/vendor/urllib3/connectionpool.py +++ b/pipenv/vendor/urllib3/connectionpool.py @@ -65,6 +65,11 @@ class ConnectionPool(object): """ Base class for all connection pools, such as :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. + + .. note:: + ConnectionPool.urlopen() does not normalize or percent-encode target URIs + which is useful if your target server doesn't support percent-encoded + target URIs. """ scheme = None @@ -760,21 +765,6 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): **response_kw ) - def drain_and_release_conn(response): - try: - # discard any remaining response body, the connection will be - # released back to the pool once the entire response is read - response.read() - except ( - TimeoutError, - HTTPException, - SocketError, - ProtocolError, - BaseSSLError, - SSLError, - ): - pass - # Handle redirect? redirect_location = redirect and response.get_redirect_location() if redirect_location: @@ -785,15 +775,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries = retries.increment(method, url, response=response, _pool=self) except MaxRetryError: if retries.raise_on_redirect: - # Drain and release the connection for this response, since - # we're not returning it to be released manually. - drain_and_release_conn(response) + response.drain_conn() raise return response - # drain and return the connection to the pool before recursing - drain_and_release_conn(response) - + response.drain_conn() retries.sleep_for_retry(response) log.debug("Redirecting %s -> %s", url, redirect_location) return self.urlopen( @@ -819,15 +805,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries = retries.increment(method, url, response=response, _pool=self) except MaxRetryError: if retries.raise_on_status: - # Drain and release the connection for this response, since - # we're not returning it to be released manually. - drain_and_release_conn(response) + response.drain_conn() raise return response - # drain and return the connection to the pool before recursing - drain_and_release_conn(response) - + response.drain_conn() retries.sleep(response) log.debug("Retry: %s", url) return self.urlopen( diff --git a/pipenv/vendor/urllib3/contrib/pyopenssl.py b/pipenv/vendor/urllib3/contrib/pyopenssl.py index 3051ef3a..81a80651 100644 --- a/pipenv/vendor/urllib3/contrib/pyopenssl.py +++ b/pipenv/vendor/urllib3/contrib/pyopenssl.py @@ -450,9 +450,12 @@ class PyOpenSSLContext(object): cafile = cafile.encode("utf-8") if capath is not None: capath = capath.encode("utf-8") - self._ctx.load_verify_locations(cafile, capath) - if cadata is not None: - self._ctx.load_verify_locations(BytesIO(cadata)) + try: + self._ctx.load_verify_locations(cafile, capath) + if cadata is not None: + self._ctx.load_verify_locations(BytesIO(cadata)) + except OpenSSL.SSL.Error as e: + raise ssl.SSLError("unable to load trusted certificates: %r" % e) def load_cert_chain(self, certfile, keyfile=None, password=None): self._ctx.use_certificate_chain_file(certfile) diff --git a/pipenv/vendor/urllib3/contrib/securetransport.py b/pipenv/vendor/urllib3/contrib/securetransport.py index 87d844af..a6b7e94a 100644 --- a/pipenv/vendor/urllib3/contrib/securetransport.py +++ b/pipenv/vendor/urllib3/contrib/securetransport.py @@ -819,6 +819,11 @@ class SecureTransportContext(object): if capath is not None: raise ValueError("SecureTransport does not support cert directories") + # Raise if cafile does not exist. + if cafile is not None: + with open(cafile): + pass + self._trust_bundle = cafile or cadata def load_cert_chain(self, certfile, keyfile=None, password=None): diff --git a/pipenv/vendor/urllib3/exceptions.py b/pipenv/vendor/urllib3/exceptions.py index 0a74c79b..5cc4d8a4 100644 --- a/pipenv/vendor/urllib3/exceptions.py +++ b/pipenv/vendor/urllib3/exceptions.py @@ -45,7 +45,10 @@ class SSLError(HTTPError): class ProxyError(HTTPError): "Raised when the connection to a proxy fails." - pass + + def __init__(self, message, error, *args): + super(ProxyError, self).__init__(message, error, *args) + self.original_error = error class DecodeError(HTTPError): @@ -195,6 +198,20 @@ class DependencyWarning(HTTPWarning): pass +class InvalidProxyConfigurationWarning(HTTPWarning): + """ + Warned when using an HTTPS proxy and an HTTPS URL. Currently + urllib3 doesn't support HTTPS proxies and the proxy will be + contacted via HTTP instead. This warning can be fixed by + changing your HTTPS proxy URL into an HTTP proxy URL. + + If you encounter this warning read this: + https://github.com/urllib3/urllib3/issues/1850 + """ + + pass + + class ResponseNotChunked(ProtocolError, ValueError): "Response needs to be chunked in order to read it as chunks." pass diff --git a/pipenv/vendor/urllib3/poolmanager.py b/pipenv/vendor/urllib3/poolmanager.py index 242a2f82..e2bd3bd8 100644 --- a/pipenv/vendor/urllib3/poolmanager.py +++ b/pipenv/vendor/urllib3/poolmanager.py @@ -2,11 +2,17 @@ from __future__ import absolute_import import collections import functools import logging +import warnings from ._collections import RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .connectionpool import port_by_scheme -from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown +from .exceptions import ( + LocationValueError, + MaxRetryError, + ProxySchemeUnknown, + InvalidProxyConfigurationWarning, +) from .packages import six from .packages.six.moves.urllib.parse import urljoin from .request import RequestMethods @@ -359,6 +365,7 @@ class PoolManager(RequestMethods): retries = retries.increment(method, url, response=response, _pool=conn) except MaxRetryError: if retries.raise_on_redirect: + response.drain_conn() raise return response @@ -366,6 +373,8 @@ class PoolManager(RequestMethods): kw["redirect"] = redirect log.info("Redirecting %s -> %s", url, redirect_location) + + response.drain_conn() return self.urlopen(method, redirect_location, **kw) @@ -452,9 +461,22 @@ class ProxyManager(PoolManager): headers_.update(headers) return headers_ + def _validate_proxy_scheme_url_selection(self, url_scheme): + if url_scheme == "https" and self.proxy.scheme == "https": + warnings.warn( + "Your proxy configuration specified an HTTPS scheme for the proxy. " + "Are you sure you want to use HTTPS to contact the proxy? " + "This most likely indicates an error in your configuration. " + "Read this issue for more info: " + "https://github.com/urllib3/urllib3/issues/1850", + InvalidProxyConfigurationWarning, + stacklevel=3, + ) + def urlopen(self, method, url, redirect=True, **kw): "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." u = parse_url(url) + self._validate_proxy_scheme_url_selection(u.scheme) if u.scheme == "http": # For proxied HTTPS requests, httplib sets the necessary headers diff --git a/pipenv/vendor/urllib3/response.py b/pipenv/vendor/urllib3/response.py index 6090a735..7dc9b93c 100644 --- a/pipenv/vendor/urllib3/response.py +++ b/pipenv/vendor/urllib3/response.py @@ -20,6 +20,7 @@ from .exceptions import ( ResponseNotChunked, IncompleteRead, InvalidHeader, + HTTPError, ) from .packages.six import string_types as basestring, PY3 from .packages.six.moves import http_client as httplib @@ -277,6 +278,17 @@ class HTTPResponse(io.IOBase): self._pool._put_conn(self._connection) self._connection = None + def drain_conn(self): + """ + Read and discard any remaining HTTP response data in the response connection. + + Unread data in the HTTPResponse connection blocks the connection from being released back to the pool. + """ + try: + self.read() + except (HTTPError, SocketError, BaseSSLError, HTTPException): + pass + @property def data(self): # For backwords-compat with earlier urllib3 0.4 and earlier. diff --git a/pipenv/vendor/urllib3/util/retry.py b/pipenv/vendor/urllib3/util/retry.py index 5a049fe6..ee30c91b 100644 --- a/pipenv/vendor/urllib3/util/retry.py +++ b/pipenv/vendor/urllib3/util/retry.py @@ -13,6 +13,7 @@ from ..exceptions import ( ReadTimeoutError, ResponseError, InvalidHeader, + ProxyError, ) from ..packages import six @@ -306,6 +307,8 @@ class Retry(object): """ Errors when we're fairly sure that the server did not receive the request, so it should be safe to retry. """ + if isinstance(err, ProxyError): + err = err.original_error return isinstance(err, ConnectTimeoutError) def _is_read_error(self, err): diff --git a/pipenv/vendor/urllib3/util/ssl_.py b/pipenv/vendor/urllib3/util/ssl_.py index 5b363d7f..f7e2b705 100644 --- a/pipenv/vendor/urllib3/util/ssl_.py +++ b/pipenv/vendor/urllib3/util/ssl_.py @@ -119,12 +119,15 @@ except ImportError: self.certfile = certfile self.keyfile = keyfile - def load_verify_locations(self, cafile=None, capath=None): + def load_verify_locations(self, cafile=None, capath=None, cadata=None): self.ca_certs = cafile if capath is not None: raise SSLError("CA directories not supported in older Pythons") + if cadata is not None: + raise SSLError("CA data not supported in older Pythons") + def set_ciphers(self, cipher_suite): self.ciphers = cipher_suite @@ -305,6 +308,7 @@ def ssl_wrap_socket( ssl_context=None, ca_cert_dir=None, key_password=None, + ca_cert_data=None, ): """ All arguments except for server_hostname, ssl_context, and ca_cert_dir have @@ -323,6 +327,9 @@ def ssl_wrap_socket( SSLContext.load_verify_locations(). :param key_password: Optional password if the keyfile is encrypted. + :param ca_cert_data: + Optional string containing CA certificates in PEM format suitable for + passing as the cadata parameter to SSLContext.load_verify_locations() """ context = ssl_context if context is None: @@ -331,9 +338,9 @@ def ssl_wrap_socket( # this code. context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers) - if ca_certs or ca_cert_dir: + if ca_certs or ca_cert_dir or ca_cert_data: try: - context.load_verify_locations(ca_certs, ca_cert_dir) + context.load_verify_locations(ca_certs, ca_cert_dir, ca_cert_data) except IOError as e: # Platform-specific: Python 2.7 raise SSLError(e) # Py33 raises FileNotFoundError which subclasses OSError diff --git a/pipenv/vendor/urllib3/util/timeout.py b/pipenv/vendor/urllib3/util/timeout.py index 98837005..b61fea75 100644 --- a/pipenv/vendor/urllib3/util/timeout.py +++ b/pipenv/vendor/urllib3/util/timeout.py @@ -98,7 +98,7 @@ class Timeout(object): self.total = self._validate_timeout(total, "total") self._start_connect = None - def __str__(self): + def __repr__(self): return "%s(connect=%r, read=%r, total=%r)" % ( type(self).__name__, self._connect, @@ -106,6 +106,9 @@ class Timeout(object): self.total, ) + # __str__ provided for backwards compatibility + __str__ = __repr__ + @classmethod def _validate_timeout(cls, value, name): """ Check that a timeout attribute is valid. diff --git a/pipenv/vendor/urllib3/util/url.py b/pipenv/vendor/urllib3/util/url.py index 8ef5a231..793324e5 100644 --- a/pipenv/vendor/urllib3/util/url.py +++ b/pipenv/vendor/urllib3/util/url.py @@ -18,7 +18,7 @@ PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}") SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)") URI_RE = re.compile( r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?" - r"(?://([^/?#]*))?" + r"(?://([^\\/?#]*))?" r"([^?#]*)" r"(?:\?([^#]*))?" r"(?:#(.*))?$", diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index dbd78deb..1a06b43b 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -24,13 +24,13 @@ pythonfinder==1.2.2 requests==2.23.0 chardet==3.0.4 idna==2.9 - urllib3==1.25.8 - certifi==2019.11.28 -requirementslib==1.5.5 + urllib3==1.25.9 + certifi==2020.4.5.1 +requirementslib==1.5.6 attrs==19.3.0 distlib==0.3.0 packaging==20.3 - pyparsing==2.4.6 + pyparsing==2.4.7 git+https://github.com/sarugaku/plette.git@master#egg=plette tomlkit==0.5.11 shellingham==1.3.2 @@ -39,19 +39,20 @@ semver==2.9.0 toml==0.10.0 cached-property==1.5.1 vistir==0.5.0 -pip-shims==0.5.1 +pip-shims==0.5.2 contextlib2==0.6.0.post1 funcsigs==1.0.2 enum34==1.1.6 # yaspin==0.15.0 yaspin==0.14.3 cerberus==1.3.2 -resolvelib==0.2.2 +resolvelib==0.3.0 backports.functools_lru_cache==1.5 -pep517==0.8.1 +pep517==0.8.2 zipp==0.6.0 - importlib_metadata==1.5.1 + importlib_metadata==1.6.0 importlib-resources==1.4.0 more-itertools==5.0.0 git+https://github.com/sarugaku/passa.git@master#egg=passa orderedmultidict==1.0.1 +dparse==0.5.0 diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index dbcdac0b..3c0cb22b 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -5,9 +5,11 @@ import io import itertools +import json import re import shutil import sys + # from tempfile import TemporaryDirectory import tarfile import zipfile @@ -23,75 +25,72 @@ from urllib3.util import parse_url as urllib3_parse from pipenv.utils import mkdir_p from pipenv.vendor.vistir.compat import NamedTemporaryFile, TemporaryDirectory from pipenv.vendor.vistir.contextmanagers import open_file +from pipenv.vendor.requirementslib.models.lockfile import Lockfile, merge_items import pipenv.vendor.parse as parse -TASK_NAME = 'update' +TASK_NAME = "update" LIBRARY_DIRNAMES = { - 'requirements-parser': 'requirements', - 'backports.shutil_get_terminal_size': 'backports/shutil_get_terminal_size', - 'backports.weakref': 'backports/weakref', - 'backports.functools_lru_cache': 'backports/functools_lru_cache', - 'python-dotenv': 'dotenv', - 'pip-tools': 'piptools', - 'setuptools': 'pkg_resources', - 'msgpack-python': 'msgpack', - 'attrs': 'attr', - 'enum': 'backports/enum' + "requirements-parser": "requirements", + "backports.shutil_get_terminal_size": "backports/shutil_get_terminal_size", + "backports.weakref": "backports/weakref", + "backports.functools_lru_cache": "backports/functools_lru_cache", + "python-dotenv": "dotenv", + "pip-tools": "piptools", + "setuptools": "pkg_resources", + "msgpack-python": "msgpack", + "attrs": "attr", + "enum": "backports/enum", } -PY2_DOWNLOAD = ['enum34'] +PY2_DOWNLOAD = ["enum34"] # from time to time, remove the no longer needed ones HARDCODED_LICENSE_URLS = { - 'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE', - 'cursor': 'https://raw.githubusercontent.com/GijsTimmers/cursor/master/LICENSE', - 'delegator.py': 'https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE', - 'click-didyoumean': 'https://raw.githubusercontent.com/click-contrib/click-didyoumean/master/LICENSE', - 'click-completion': 'https://raw.githubusercontent.com/click-contrib/click-completion/master/LICENSE', - 'parse': 'https://raw.githubusercontent.com/techalchemy/parse/master/LICENSE', - 'semver': 'https://raw.githubusercontent.com/k-bx/python-semver/master/LICENSE.txt', - 'crayons': 'https://raw.githubusercontent.com/kennethreitz/crayons/master/LICENSE', - 'pip-tools': 'https://raw.githubusercontent.com/jazzband/pip-tools/master/LICENSE', - 'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE', - 'webencodings': 'https://github.com/SimonSapin/python-webencodings/raw/' - 'master/LICENSE', - 'requirementslib': 'https://github.com/techalchemy/requirementslib/raw/master/LICENSE', - 'distlib': 'https://github.com/vsajip/distlib/raw/master/LICENSE.txt', - 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt', - 'pyparsing': 'https://raw.githubusercontent.com/pyparsing/pyparsing/master/LICENSE', - 'resolvelib': 'https://raw.githubusercontent.com/sarugaku/resolvelib/master/LICENSE', - 'funcsigs': 'https://raw.githubusercontent.com/aliles/funcsigs/master/LICENSE' + "pytoml": "https://github.com/avakar/pytoml/raw/master/LICENSE", + "cursor": "https://raw.githubusercontent.com/GijsTimmers/cursor/master/LICENSE", + "delegator.py": "https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE", + "click-didyoumean": "https://raw.githubusercontent.com/click-contrib/click-didyoumean/master/LICENSE", + "click-completion": "https://raw.githubusercontent.com/click-contrib/click-completion/master/LICENSE", + "parse": "https://raw.githubusercontent.com/techalchemy/parse/master/LICENSE", + "semver": "https://raw.githubusercontent.com/k-bx/python-semver/master/LICENSE.txt", + "crayons": "https://raw.githubusercontent.com/kennethreitz/crayons/master/LICENSE", + "pip-tools": "https://raw.githubusercontent.com/jazzband/pip-tools/master/LICENSE", + "pytoml": "https://github.com/avakar/pytoml/raw/master/LICENSE", + "webencodings": "https://github.com/SimonSapin/python-webencodings/raw/" + "master/LICENSE", + "requirementslib": "https://github.com/techalchemy/requirementslib/raw/master/LICENSE", + "distlib": "https://github.com/vsajip/distlib/raw/master/LICENSE.txt", + "pythonfinder": "https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt", + "pyparsing": "https://raw.githubusercontent.com/pyparsing/pyparsing/master/LICENSE", + "resolvelib": "https://raw.githubusercontent.com/sarugaku/resolvelib/master/LICENSE", + "funcsigs": "https://raw.githubusercontent.com/aliles/funcsigs/master/LICENSE", } FILE_WHITE_LIST = ( - 'Makefile', - 'vendor.txt', - 'patched.txt', - '__init__.py', - 'README.rst', - 'README.md', - 'appdirs.py', - 'safety.zip', - 'cacert.pem', - 'vendor_pip.txt', + "Makefile", + "vendor.txt", + "patched.txt", + "__init__.py", + "README.rst", + "README.md", + "appdirs.py", + "safety.zip", + "cacert.pem", + "vendor_pip.txt", ) -PATCHED_RENAMES = { - 'pip': 'notpip' -} +PATCHED_RENAMES = {"pip": "notpip"} LIBRARY_RENAMES = { - 'pip': 'pipenv.patched.notpip', + "pip": "pipenv.patched.notpip", "functools32": "pipenv.vendor.backports.functools_lru_cache", - 'enum34': 'enum', + "enum34": "enum", } -LICENSE_RENAMES = { - "pythonfinder/LICENSE": "pythonfinder/pep514tools.LICENSE" -} +LICENSE_RENAMES = {"pythonfinder/LICENSE": "pythonfinder/pep514tools.LICENSE"} def drop_dir(path): @@ -108,32 +107,32 @@ def remove_all(paths): def log(msg): - print('[vendoring.%s] %s' % (TASK_NAME, msg)) + print("[vendoring.%s] %s" % (TASK_NAME, msg)) def _get_git_root(ctx): - return Path(ctx.run('git rev-parse --show-toplevel', hide=True).stdout.strip()) + return Path(ctx.run("git rev-parse --show-toplevel", hide=True).stdout.strip()) def _get_vendor_dir(ctx): - return _get_git_root(ctx) / 'pipenv' / 'vendor' + return _get_git_root(ctx) / "pipenv" / "vendor" def _get_patched_dir(ctx): - return _get_git_root(ctx) / 'pipenv' / 'patched' + return _get_git_root(ctx) / "pipenv" / "patched" def clean_vendor(ctx, vendor_dir): # Old _vendor cleanup - remove_all(vendor_dir.glob('*.pyc')) - log('Cleaning %s' % vendor_dir) + remove_all(vendor_dir.glob("*.pyc")) + log("Cleaning %s" % vendor_dir) for item in vendor_dir.iterdir(): if item.is_dir(): shutil.rmtree(str(item)) elif item.name not in FILE_WHITE_LIST: item.unlink() else: - log('Skipping %s' % item) + log("Skipping %s" % item) def detect_vendored_libs(vendor_dir): @@ -154,7 +153,7 @@ def rewrite_imports(package_dir, vendored_libs, vendor_dir): for item in package_dir.iterdir(): if item.is_dir(): rewrite_imports(item, vendored_libs, vendor_dir) - elif item.name.endswith('.py'): + elif item.name.endswith(".py"): rewrite_file_imports(item, vendored_libs, vendor_dir) @@ -162,9 +161,9 @@ def rewrite_file_imports(item, vendored_libs, vendor_dir): """Rewrite 'import xxx' and 'from xxx import' for vendored_libs""" # log('Reading file: %s' % item) try: - text = item.read_text(encoding='utf-8') + text = item.read_text(encoding="utf-8") except UnicodeDecodeError: - text = item.read_text(encoding='cp1252') + text = item.read_text(encoding="cp1252") renames = LIBRARY_RENAMES for k in LIBRARY_RENAMES.keys(): if k not in vendored_libs: @@ -174,111 +173,165 @@ def rewrite_file_imports(item, vendored_libs, vendor_dir): if lib in renames: to_lib = renames[lib] text = re.sub( - r'([\n\s]*)import %s([\n\s\.]+)' % lib, - r'\1import %s\2' % to_lib, - text, - ) - text = re.sub( - r'([\n\s]*)from %s([\s\.])+' % lib, - r'\1from %s\2' % to_lib, - text, + r"([\n\s]*)import %s([\n\s\.]+)" % lib, r"\1import %s\2" % to_lib, text, ) + text = re.sub(r"([\n\s]*)from %s([\s\.])+" % lib, r"\1from %s\2" % to_lib, text,) text = re.sub( r"(\n\s*)__import__\('%s([\s'\.])+" % lib, r"\1__import__('%s\2" % to_lib, text, ) - item.write_text(text, encoding='utf-8') + item.write_text(text, encoding="utf-8") def apply_patch(ctx, patch_file_path): - log('Applying patch %s' % patch_file_path.name) - ctx.run('git apply --ignore-whitespace --verbose %s' % patch_file_path) + log("Applying patch %s" % patch_file_path.name) + ctx.run("git apply --ignore-whitespace --verbose %s" % patch_file_path) + + +def _recursive_write_to_zip(zf, path, root=None): + if path == Path(zf.filename): + return + if root is None: + if not path.is_dir(): + raise ValueError('root is required for non-directory path') + root = path + if not path.is_dir(): + zf.write(str(path), str(path.relative_to(root))) + return + for c in path.iterdir(): + _recursive_write_to_zip(zf, c, root) @invoke.task def update_safety(ctx): - ignore_subdeps = ['pip', 'pip-egg-info', 'bin'] - ignore_files = ['pip-delete-this-directory.txt', 'PKG-INFO'] - vendor_dir = _get_patched_dir(ctx) - log('Using vendor dir: %s' % vendor_dir) - log('Downloading safety package files...') - build_dir = vendor_dir / 'build' - download_dir = TemporaryDirectory(prefix='pipenv-', suffix='-safety') - if build_dir.exists() and build_dir.is_dir(): - drop_dir(build_dir) - - ctx.run( - 'pip download -b {0} --no-binary=:all: --no-clean -d {1} safety pyyaml'.format( - str(build_dir), str(download_dir.name), + ignore_subdeps = ["pip", "pip-egg-info", "bin", "pipenv", "virtualenv", "virtualenv-clone", "setuptools",] + ignore_files = ["pip-delete-this-directory.txt", "PKG-INFO", "easy_install.py", "clonevirtualenv.py"] + ignore_patterns = ["*.pyd", "*.so", "**/*.pyc", "*.pyc"] + cmd_envvars = { + "PIPENV_NO_INHERIT": "true", + "PIPENV_IGNORE_VIRTUALENVS": "true", + "PIPENV_VENV_IN_PROJECT": "true" + } + patched_dir = _get_patched_dir(ctx) + vendor_dir = _get_vendor_dir(ctx) + safety_dir = Path(__file__).absolute().parent.joinpath("safety") + log("Using vendor dir: %s" % patched_dir) + log("Downloading safety package files...") + build_dir = patched_dir / "build" + root = _get_git_root(ctx) + with TemporaryDirectory(prefix="pipenv-", suffix="-safety") as download_dir: + log("generating lockfile...") + packages = "\n".join(["safety", "requests[security]"]) + env = {"PIPENV_PACKAGES": packages} + resolve_cmd = "python {0}".format(root.joinpath("pipenv/resolver.py").as_posix()) + py27_resolve_cmd = "python2.7 {0}".format(root.joinpath("pipenv/resolver.py").as_posix()) + _, _, resolved = ctx.run(resolve_cmd, hide=True, env=env).stdout.partition("RESULTS:") + _, _, resolved_py2 = ctx.run(py27_resolve_cmd, hide=True, env=env).stdout.partition("RESULTS:") + resolved = json.loads(resolved.strip()) + resolved_py2 = json.loads(resolved_py2.strip()) + pkg_dict, pkg_dict_py2 = {}, {} + for pkg in resolved: + name = pkg.pop("name") + pkg["version"] = "=={0}".format(pkg["version"]) + pkg_dict[name] = pkg + for pkg in resolved_py2: + name = pkg.pop("name") + pkg["version"] = "=={0}".format(pkg["version"]) + pkg_dict_py2[name] = pkg + merged = merge_items([pkg_dict, pkg_dict_py2]) + lf = Lockfile.create(safety_dir.as_posix()) + lf["default"] = merged + lf.write() + # envvars_no_deps = {"PIP_NO_DEPS": "true"}.update(cmd_envvars) + # ctx.run("python -m pipenv run pip install safety", env=envvars_no_deps) + # ctx.run("python -m pipenv run pip uninstall -y pipenv", env=cmd_envvars) + # ctx.run("python -m pipenv install safety", env=cmd_envvars) + # ctx.run("python -m pipenv run pip uninstall -y pipenv", env=cmd_envvars) + # ctx.run("python2.7 -m pip install --upgrade --upgrade-strategy=eager -e {}".format(root.as_posix())) + # ctx.run("python2.7 -m pipenv install safety", env=cmd_envvars) + # requirements_txt = ctx.run("python2.7 -m pipenv lock -r", env=cmd_envvars, quiet=True).out + requirements = [ + r.as_line(include_hashes=False, include_markers=False) + for r in lf.requirements + ] + safety_dir.joinpath("requirements.txt").write_text("\n".join(requirements)) + if build_dir.exists() and build_dir.is_dir(): + log("dropping pre-existing build dir at {0}".format(build_dir.as_posix())) + drop_dir(build_dir) + pip_command = "pip download -b {0} --no-binary=:all: --no-clean --no-deps -d {1} pyyaml safety".format( + build_dir.absolute().as_posix(), str(download_dir.name), ) - ) - safety_dir = build_dir / 'safety' - yaml_build_dir = build_dir / 'pyyaml' - main_file = safety_dir / '__main__.py' - main_content = """ -import sys -yaml_lib = 'yaml{0}'.format(sys.version_info[0]) -locals()[yaml_lib] = __import__(yaml_lib) -sys.modules['yaml'] = sys.modules[yaml_lib] -from safety.cli import cli + log("downloading deps via pip: {0}".format(pip_command)) + ctx.run(pip_command) + safety_build_dir = build_dir / "safety" + yaml_build_dir = build_dir / "pyyaml" + lib_dir = safety_dir.joinpath("lib") -# Disable insecure warnings. -import urllib3 -from urllib3.exceptions import InsecureRequestWarning -urllib3.disable_warnings(InsecureRequestWarning) - -cli(prog_name="safety") - """.strip() - with open(str(main_file), 'w') as fh: - fh.write(main_content) - - with ctx.cd(str(safety_dir)): - ctx.run('pip install --no-compile --no-binary=:all: -t . .') - safety_dir = safety_dir.absolute() - yaml_dir = safety_dir / 'yaml' - if yaml_dir.exists(): - version_choices = ['2', '3'] - version_choices.remove(str(sys.version_info[0])) - mkdir_p(str(safety_dir / 'yaml{0}'.format(sys.version_info[0]))) - for fn in yaml_dir.glob('*.py'): - fn.rename(str(safety_dir.joinpath('yaml{0}'.format(sys.version_info[0]), fn.name))) - if version_choices[0] == '2': - lib = yaml_build_dir / 'lib' / 'yaml' - else: - lib = yaml_build_dir / 'lib3' / 'yaml' - shutil.copytree(str(lib.absolute()), str(safety_dir / 'yaml{0}'.format(version_choices[0]))) - requests_dir = safety_dir / 'requests' - cacert = vendor_dir / 'requests' / 'cacert.pem' - if not cacert.exists(): - from pipenv.vendor import requests - cacert = Path(requests.certs.where()) - target_cert = requests_dir / 'cacert.pem' - target_cert.write_bytes(cacert.read_bytes()) - ctx.run("sed -i 's/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers)/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, verify=False)/g' {0}".format(str(safety_dir / 'safety' / 'safety.py'))) - for egg in safety_dir.glob('*.egg-info'): - drop_dir(egg.absolute()) - for dep in ignore_subdeps: - dep_dir = safety_dir / dep - if dep_dir.exists(): - drop_dir(dep_dir) - for dep in ignore_files: - fn = safety_dir / dep - if fn.exists(): - fn.unlink() - zip_name = '{0}/safety'.format(str(vendor_dir)) - shutil.make_archive(zip_name, format='zip', root_dir=str(safety_dir), base_dir='./') - drop_dir(build_dir) - download_dir.cleanup() + with ctx.cd(str(safety_dir)): + lib_dir.mkdir(exist_ok=True) + install_cmd = "python2.7 -m pip install --ignore-requires-python -t {0} -r {1}".format(lib_dir.as_posix(), safety_dir.joinpath("requirements.txt").as_posix()) + log("installing dependencies: {0}".format(install_cmd)) + ctx.run(install_cmd) + safety_dir = safety_dir.absolute() + yaml_dir = lib_dir / "yaml" + yaml_lib_dir_map = { + "2": { + "current_path": yaml_build_dir / "lib/yaml", + "destination": lib_dir / "yaml2", + }, + "3": { + "current_path": yaml_build_dir / "lib3/yaml", + "destination": lib_dir / "yaml3", + }, + } + if yaml_dir.exists(): + drop_dir(yaml_dir) + log("Mapping yaml paths for python 2 and 3...") + for py_version, path_dict in yaml_lib_dir_map.items(): + path_dict["current_path"].rename(path_dict["destination"]) + log("Ensuring certificates are available...") + requests_dir = lib_dir / "requests" + cacert = vendor_dir / "certifi" / "cacert.pem" + if not cacert.exists(): + from pipenv.vendor import requests + cacert = Path(requests.certs.where()) + target_cert = requests_dir / "cacert.pem" + target_cert.write_bytes(cacert.read_bytes()) + log("dropping ignored files...") + for pattern in ignore_patterns: + for path in lib_dir.rglob(pattern): + log("removing {0!s}".format(path)) + path.unlink() + for dep in ignore_subdeps: + if lib_dir.joinpath(dep).exists(): + log("cleaning up {0}".format(dep)) + drop_dir(lib_dir.joinpath(dep)) + for path in itertools.chain.from_iterable(( + lib_dir.rglob("{0}*.egg-info".format(dep)), + lib_dir.rglob("{0}*.dist-info".format(dep)) + )): + log("cleaning up {0}".format(path)) + drop_dir(path) + for fn in ignore_files: + for path in lib_dir.rglob(fn): + log("cleaning up {0}".format(path)) + path.unlink() + zip_name = "{0}/safety.zip".format(str(patched_dir)) + log("writing zipfile...") + with zipfile.ZipFile(zip_name, 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=6) as zf: + _recursive_write_to_zip(zf, safety_dir) + drop_dir(build_dir) + drop_dir(lib_dir) def rename_if_needed(ctx, vendor_dir, item): - rename_dict = LIBRARY_RENAMES if vendor_dir.name != 'patched' else PATCHED_RENAMES + rename_dict = LIBRARY_RENAMES if vendor_dir.name != "patched" else PATCHED_RENAMES new_path = None if item.name in rename_dict or item.name in LIBRARY_DIRNAMES: new_name = rename_dict.get(item.name, LIBRARY_DIRNAMES.get(item.name)) new_path = item.parent / new_name - log('Renaming %s => %s' % (item.name, new_path)) + log("Renaming %s => %s" % (item.name, new_path)) # handle existing directories try: item.rename(str(new_path)) @@ -288,55 +341,89 @@ def rename_if_needed(ctx, vendor_dir, item): def write_backport_imports(ctx, vendor_dir): - backport_dir = vendor_dir / 'backports' + backport_dir = vendor_dir / "backports" if not backport_dir.exists(): return - backport_init = backport_dir / '__init__.py' + backport_init = backport_dir / "__init__.py" backport_libs = detect_vendored_libs(backport_dir) init_py_lines = backport_init.read_text().splitlines() for lib in backport_libs: - lib_line = 'from . import {0}'.format(lib) + lib_line = "from . import {0}".format(lib) if lib_line not in init_py_lines: - log('Adding backport %s to __init__.py exports' % lib) + log("Adding backport %s to __init__.py exports" % lib) init_py_lines.append(lib_line) - backport_init.write_text('\n'.join(init_py_lines) + '\n') + backport_init.write_text("\n".join(init_py_lines) + "\n") def _ensure_package_in_requirements(ctx, requirements_file, package): requirement = None - log('using requirements file: %s' % requirements_file) + log("using requirements file: %s" % requirements_file) req_file_lines = [l for l in requirements_file.read_text().splitlines()] if package: match = [r for r in req_file_lines if r.strip().lower().startswith(package)] matched_req = None if match: for m in match: - specifiers = [m.index(s) for s in ['>', '<', '=', '~'] if s in m] - if m.lower() == package or (specifiers and m[:min(specifiers)].lower() == package): + specifiers = [m.index(s) for s in [">", "<", "=", "~"] if s in m] + if m.lower() == package or ( + specifiers and m[: min(specifiers)].lower() == package + ): matched_req = "{0}".format(m) requirement = matched_req log("Matched req: %r" % matched_req) if not matched_req: req_file_lines.append("{0}".format(package)) log("Writing requirements file: %s" % requirements_file) - requirements_file.write_text('\n'.join(req_file_lines)) + requirements_file.write_text("\n".join(req_file_lines)) requirement = "{0}".format(package) return requirement +def install_pyyaml(ctx, vendor_dir): + build_dir = vendor_dir / "build" + if build_dir.exists() and build_dir.is_dir(): + log("dropping pre-existing build dir at {0}".format(build_dir.as_posix())) + drop_dir(build_dir) + with TemporaryDirectory(prefix="pipenv-", suffix="-safety") as download_dir: + pip_command = "pip download -b {0} --no-binary=:all: --no-clean --no-deps -d {1} pyyaml safety".format( + build_dir.absolute().as_posix(), str(download_dir.name), + ) + log("downloading deps via pip: {0}".format(pip_command)) + ctx.run(pip_command) + safety_build_dir = build_dir / "safety" + yaml_build_dir = build_dir / "pyyaml" + yaml_dir = vendor_dir / "yaml" + yaml_lib_dir_map = { + "2": { + "current_path": yaml_build_dir / "lib/yaml", + "destination": vendor_dir / "yaml2", + }, + "3": { + "current_path": yaml_build_dir / "lib3/yaml", + "destination": vendor_dir / "yaml3", + }, + } + if yaml_dir.exists(): + drop_dir(yaml_dir) + log("Mapping yaml paths for python 2 and 3...") + for py_version, path_dict in yaml_lib_dir_map.items(): + path_dict["current_path"].rename(path_dict["destination"]) + path_dict["destination"].joinpath("LICENSE").write_text(yaml_build_dir.joinpath("LICENSE").read_text()) + drop_dir(build_dir) + + def install(ctx, vendor_dir, package=None): requirements_file = vendor_dir / "{0}.txt".format(vendor_dir.name) requirement = "-r {0}".format(requirements_file.as_posix()) - log('Using requirements file: %s' % requirement) + log("Using requirements file: %s" % requirement) if package: requirement = _ensure_package_in_requirements(ctx, requirements_file, package) # We use --no-deps because we want to ensure that all of our dependencies # are added to vendor.txt, this includes all dependencies recursively up # the chain. ctx.run( - 'pip install -t {0} --no-compile --no-deps --upgrade {1}'.format( - vendor_dir.as_posix(), - requirement, + "pip install -t {0} --no-compile --no-deps --upgrade {1}".format( + vendor_dir.as_posix(), requirement, ) ) # read licenses from distinfo files if possible @@ -346,24 +433,32 @@ def install(ctx, vendor_dir, package=None): if not license_file.exists(): continue if vendor_dir.joinpath(pkg).exists(): - vendor_dir.joinpath(pkg).joinpath("LICENSE").write_text(license_file.read_text()) + vendor_dir.joinpath(pkg).joinpath("LICENSE").write_text( + license_file.read_text() + ) elif vendor_dir.joinpath("{0}.py".format(pkg)).exists(): - vendor_dir.joinpath("{0}.py.LICENSE".format(pkg)).write_text(license_file.read_text()) + vendor_dir.joinpath("{0}.py.LICENSE".format(pkg)).write_text( + license_file.read_text() + ) else: - matched_path = next(iter(pth for pth in vendor_dir.glob("{0}*".format(pkg))), None) + matched_path = next( + iter(pth for pth in vendor_dir.glob("{0}*".format(pkg))), None + ) if matched_path is not None: - vendor_dir.joinpath("{0}.LICENSE".format(matched_path)).write_text(license_file.read_text()) + vendor_dir.joinpath("{0}.LICENSE".format(matched_path)).write_text( + license_file.read_text() + ) def post_install_cleanup(ctx, vendor_dir): - remove_all(vendor_dir.glob('*.dist-info')) - remove_all(vendor_dir.glob('*.egg-info')) + remove_all(vendor_dir.glob("*.dist-info")) + remove_all(vendor_dir.glob("*.egg-info")) # Cleanup setuptools unneeded parts - drop_dir(vendor_dir / 'bin') - drop_dir(vendor_dir / 'tests') - drop_dir(vendor_dir / 'shutil_backports') - remove_all(vendor_dir.glob('toml.py')) + drop_dir(vendor_dir / "bin") + drop_dir(vendor_dir / "tests") + drop_dir(vendor_dir / "shutil_backports") + remove_all(vendor_dir.glob("toml.py")) @invoke.task @@ -373,24 +468,24 @@ def apply_patches(ctx, patched=False, pre=False): else: vendor_dir = _get_vendor_dir(ctx) log("Applying pre-patches...") - patch_dir = Path(__file__).parent / 'patches' / vendor_dir.name + patch_dir = Path(__file__).parent / "patches" / vendor_dir.name if pre: if not patched: pass - for patch in patch_dir.glob('*.patch'): - if not patch.name.startswith('_post'): + for patch in patch_dir.glob("*.patch"): + if not patch.name.startswith("_post"): apply_patch(ctx, patch) else: - patches = patch_dir.glob('*.patch' if not patched else '_post*.patch') + patches = patch_dir.glob("*.patch" if not patched else "_post*.patch") for patch in patches: apply_patch(ctx, patch) def vendor(ctx, vendor_dir, package=None, rewrite=True): - log('Reinstalling vendored libraries') - is_patched = vendor_dir.name == 'patched' + log("Reinstalling vendored libraries") + is_patched = vendor_dir.name == "patched" install(ctx, vendor_dir, package=package) - log('Running post-install cleanup...') + log("Running post-install cleanup...") post_install_cleanup(ctx, vendor_dir) # Detect the vendored packages/modules vendored_libs = detect_vendored_libs(_get_vendor_dir(ctx)) @@ -401,18 +496,17 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): if is_patched: apply_patches(ctx, patched=is_patched, pre=True) log("Removing scandir library files...") - remove_all(vendor_dir.glob('*.so')) - drop_dir(vendor_dir / 'setuptools') - drop_dir(vendor_dir / 'pkg_resources' / '_vendor') - drop_dir(vendor_dir / 'pkg_resources' / 'extern') - drop_dir(vendor_dir / 'bin') + for extension in ("*.so", "*.pyd", "*.egg-info", "*.dist-info"): + remove_all(vendor_dir.glob(extension)) + for dirname in ("setuptools", "pkg_resources/_vendor", "pkg_resources/extern", "bin"): + drop_dir(vendor_dir / dirname) # Global import rewrites - log('Renaming specified libs...') + log("Renaming specified libs...") for item in vendor_dir.iterdir(): if item.is_dir(): if rewrite and not package or (package and item.name.lower() in package): - log('Rewriting imports for %s...' % item) + log("Rewriting imports for %s..." % item) rewrite_imports(item, vendored_libs, vendor_dir) rename_if_needed(ctx, vendor_dir, item) elif item.name not in FILE_WHITE_LIST: @@ -422,23 +516,23 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): if not package: apply_patches(ctx, patched=is_patched, pre=False) if is_patched: - piptools_vendor = vendor_dir / 'piptools' / '_vendored' + piptools_vendor = vendor_dir / "piptools" / "_vendored" if piptools_vendor.exists(): drop_dir(piptools_vendor) - msgpack = vendor_dir / 'notpip' / '_vendor' / 'msgpack' + msgpack = vendor_dir / "notpip" / "_vendor" / "msgpack" if msgpack.exists(): - remove_all(msgpack.glob('*.so')) + remove_all(msgpack.glob("*.so")) @invoke.task def redo_imports(ctx, library): vendor_dir = _get_vendor_dir(ctx) - log('Using vendor dir: %s' % vendor_dir) + log("Using vendor dir: %s" % vendor_dir) vendored_libs = detect_vendored_libs(vendor_dir) item = vendor_dir / library - library_name = vendor_dir / '{0}.py'.format(library) + library_name = vendor_dir / "{0}.py".format(library) log("Detected vendored libraries: %s" % ", ".join(vendored_libs)) - log('Rewriting imports for %s...' % item) + log("Rewriting imports for %s..." % item) if item.is_dir(): rewrite_imports(item, vendored_libs, vendor_dir) else: @@ -448,7 +542,7 @@ def redo_imports(ctx, library): @invoke.task def rewrite_all_imports(ctx): vendor_dir = _get_vendor_dir(ctx) - log('Using vendor dir: %s' % vendor_dir) + log("Using vendor dir: %s" % vendor_dir) vendored_libs = detect_vendored_libs(vendor_dir) log("Detected vendored libraries: %s" % ", ".join(vendored_libs)) log("Rewriting all imports related to vendored libs") @@ -460,16 +554,21 @@ def rewrite_all_imports(ctx): @invoke.task -def packages_missing_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', package=None): +def packages_missing_licenses( + ctx, vendor_dir=None, requirements_file="vendor.txt", package=None +): if not vendor_dir: vendor_dir = _get_vendor_dir(ctx) requirements = vendor_dir.joinpath(requirements_file).read_text().splitlines() new_requirements = [] LICENSE_EXTS = ("rst", "txt", "APACHE", "BSD", "md") - LICENSES = [".".join(lic) for lic in itertools.product(("LICENSE", "LICENSE-MIT"), LICENSE_EXTS)] + LICENSES = [ + ".".join(lic) + for lic in itertools.product(("LICENSE", "LICENSE-MIT"), LICENSE_EXTS) + ] for i, req in enumerate(requirements): pkg = req.strip().split("=")[0] - possible_pkgs = [pkg, pkg.replace('-', '_')] + possible_pkgs = [pkg, pkg.replace("-", "_")] match_found = False if pkg in PY2_DOWNLOAD: match_found = True @@ -506,29 +605,37 @@ def packages_missing_licenses(ctx, vendor_dir=None, requirements_file='vendor.tx @invoke.task def download_licenses( - ctx, vendor_dir=None, requirements_file='vendor.txt', package=None, only=False, - patched=False + ctx, + vendor_dir=None, + requirements_file="vendor.txt", + package=None, + only=False, + patched=False, ): - log('Downloading licenses') + log("Downloading licenses") if not vendor_dir: if patched: vendor_dir = _get_patched_dir(ctx) - requirements_file = 'patched.txt' + requirements_file = "patched.txt" else: vendor_dir = _get_vendor_dir(ctx) requirements_file = vendor_dir / requirements_file - requirements = packages_missing_licenses(ctx, vendor_dir, requirements_file, package=package) + requirements = packages_missing_licenses( + ctx, vendor_dir, requirements_file, package=package + ) - with NamedTemporaryFile(prefix="pipenv", suffix="vendor-reqs", delete=False, mode="w") as fh: + with NamedTemporaryFile( + prefix="pipenv", suffix="vendor-reqs", delete=False, mode="w" + ) as fh: fh.write("\n".join(requirements)) new_requirements_file = fh.name new_requirements_file = Path(new_requirements_file) log(requirements) - tmp_dir = vendor_dir / '__tmp__' + tmp_dir = vendor_dir / "__tmp__" # TODO: Fix this whenever it gets sorted out (see https://github.com/pypa/pip/issues/5739) cmd = "pip download --no-binary :all: --only-binary requests_download --no-deps" enum_cmd = "pip download --no-deps" - ctx.run('pip install flit') # needed for the next step + ctx.run("pip install flit") # needed for the next step for req in requirements_file.read_text().splitlines(): if req.startswith("enum34"): exe_cmd = "{0} -d {1} {2}".format(enum_cmd, tmp_dir.as_posix(), req) @@ -553,9 +660,7 @@ def download_licenses( backend, _, _ = backend.partition(".") ctx.run("pip install {0}".format(backend)) ctx.run( - "{0} --no-build-isolation -d {1} {2}".format( - cmd, tmp_dir.as_posix(), req - ) + "{0} --no-build-isolation -d {1} {2}".format(cmd, tmp_dir.as_posix(), req) ) for sdist in tmp_dir.iterdir(): extract_license(vendor_dir, sdist) @@ -564,18 +669,18 @@ def download_licenses( def extract_license(vendor_dir, sdist): - if sdist.stem.endswith('.tar'): + if sdist.stem.endswith(".tar"): ext = sdist.suffix[1:] - with tarfile.open(sdist, mode='r:{}'.format(ext)) as tar: + with tarfile.open(sdist, mode="r:{}".format(ext)) as tar: found = find_and_extract_license(vendor_dir, tar, tar.getmembers()) - elif sdist.suffix in ('.zip', '.whl'): + elif sdist.suffix in (".zip", ".whl"): with zipfile.ZipFile(sdist) as zip: found = find_and_extract_license(vendor_dir, zip, zip.infolist()) else: - raise NotImplementedError('new sdist type!') + raise NotImplementedError("new sdist type!") if not found: - log('License not found in {}, will download'.format(sdist.name)) + log("License not found in {}, will download".format(sdist.name)) license_fallback(vendor_dir, sdist.name) @@ -586,10 +691,10 @@ def find_and_extract_license(vendor_dir, tar, members): name = member.name except AttributeError: # zipfile name = member.filename - if 'LICENSE' in name or 'COPYING' in name: - if '/test' in name: + if "LICENSE" in name or "COPYING" in name: + if "/test" in name: # some testing licenses in hml5lib and distlib - log('Ignoring {}'.format(name)) + log("Ignoring {}".format(name)) continue found = True extract_license_member(vendor_dir, tar, member, name) @@ -600,13 +705,13 @@ def license_fallback(vendor_dir, sdist_name): """Hardcoded license URLs. Check when updating if those are still needed""" libname = libname_from_dir(sdist_name) if libname not in HARDCODED_LICENSE_URLS: - raise ValueError('No hardcoded URL for {} license'.format(libname)) + raise ValueError("No hardcoded URL for {} license".format(libname)) url = HARDCODED_LICENSE_URLS[libname] - _, _, name = url.rpartition('/') + _, _, name = url.rpartition("/") dest = license_destination(vendor_dir, libname, name) r = requests.get(url, allow_redirects=True) - log('Downloading {}'.format(url)) + log("Downloading {}".format(url)) r.raise_for_status() dest.write_bytes(r.content) @@ -614,11 +719,11 @@ def license_fallback(vendor_dir, sdist_name): def libname_from_dir(dirname): """Reconstruct the library name without it's version""" parts = [] - for part in dirname.split('-'): + for part in dirname.split("-"): if part[0].isdigit(): break parts.append(part) - return '-'.join(parts) + return "-".join(parts) def license_destination(vendor_dir, libname, filename): @@ -626,10 +731,10 @@ def license_destination(vendor_dir, libname, filename): normal = vendor_dir / libname if normal.is_dir(): return normal / filename - lowercase = vendor_dir / libname.lower().replace('-', '_') + lowercase = vendor_dir / libname.lower().replace("-", "_") if lowercase.is_dir(): return lowercase / filename - rename_dict = LIBRARY_RENAMES if vendor_dir.name != 'patched' else PATCHED_RENAMES + rename_dict = LIBRARY_RENAMES if vendor_dir.name != "patched" else PATCHED_RENAMES # Short circuit all logic if we are renaming the whole library if libname in rename_dict: return vendor_dir / rename_dict[libname] / filename @@ -637,15 +742,15 @@ def license_destination(vendor_dir, libname, filename): override = vendor_dir / LIBRARY_DIRNAMES[libname] if not override.exists() and override.parent.exists(): # for flattened subdeps, specifically backports/weakref.py - return ( - vendor_dir / override.parent - ) / '{0}.{1}'.format(override.name, filename) + return (vendor_dir / override.parent) / "{0}.{1}".format( + override.name, filename + ) license_path = Path(LIBRARY_DIRNAMES[libname]) / filename if license_path.as_posix() in LICENSE_RENAMES: return vendor_dir / LICENSE_RENAMES[license_path.as_posix()] return vendor_dir / LIBRARY_DIRNAMES[libname] / filename # fallback to libname.LICENSE (used for nondirs) - return vendor_dir / '{}.{}'.format(libname, filename) + return vendor_dir / "{}.{}".format(libname, filename) def extract_license_member(vendor_dir, tar, member, name): @@ -653,7 +758,7 @@ def extract_license_member(vendor_dir, tar, member, name): dirname = list(mpath.parents)[-2].name # -1 is . libname = libname_from_dir(dirname) dest = license_destination(vendor_dir, libname, mpath.name) - log('Extracting {} into {}'.format(name, dest)) + log("Extracting {} into {}".format(name, dest)) try: fileobj = tar.extractfile(member) dest.write_bytes(fileobj.read()) @@ -662,18 +767,20 @@ def extract_license_member(vendor_dir, tar, member, name): @invoke.task() -def generate_patch(ctx, package_path, patch_description, base='HEAD'): +def generate_patch(ctx, package_path, patch_description, base="HEAD"): pkg = Path(package_path) - if len(pkg.parts) != 2 or pkg.parts[0] not in ('vendor', 'patched'): - raise ValueError('example usage: generate-patch patched/piptools some-description') + if len(pkg.parts) != 2 or pkg.parts[0] not in ("vendor", "patched"): + raise ValueError( + "example usage: generate-patch patched/piptools some-description" + ) if patch_description: - patch_fn = '{0}-{1}.patch'.format(pkg.parts[1], patch_description) + patch_fn = "{0}-{1}.patch".format(pkg.parts[1], patch_description) else: - patch_fn = '{0}.patch'.format(pkg.parts[1]) - command = 'git diff {base} -p {root} > {out}'.format( + patch_fn = "{0}.patch".format(pkg.parts[1]) + command = "git diff {base} -p {root} > {out}".format( base=base, - root=Path('pipenv').joinpath(pkg), - out=Path(__file__).parent.joinpath('patches', pkg.parts[0], patch_fn), + root=Path("pipenv").joinpath(pkg), + out=Path(__file__).parent.joinpath("patches", pkg.parts[0], patch_fn), ) with ctx.cd(str(_get_git_root(ctx))): log(command) @@ -717,14 +824,20 @@ def unpin_and_copy_requirements(ctx, requirement_file, name="requirements.txt"): target = Path(tempdir.name).joinpath("requirements.txt") contents = unpin_file(requirement_file.read_text()) target.write_text(contents) - env = {"PIPENV_IGNORE_VIRTUALENVS": "1", "PIPENV_NOSPIN": "1", "PIPENV_PYTHON": "2.7"} + env = { + "PIPENV_IGNORE_VIRTUALENVS": "1", + "PIPENV_NOSPIN": "1", + "PIPENV_PYTHON": "2.7", + } with ctx.cd(tempdir.name): ctx.run("pipenv install -r {0}".format(target.as_posix()), env=env, hide=True) result = ctx.run("pipenv lock -r", env=env, hide=True).stdout.strip() ctx.run("pipenv --rm", env=env, hide=True) result = list(sorted([line.strip() for line in result.splitlines()[1:]])) new_requirements = requirement_file.parent.joinpath(name) - requirement_file.rename(requirement_file.parent.joinpath("{}.bak".format(name))) + requirement_file.rename( + requirement_file.parent.joinpath("{}.bak".format(name)) + ) new_requirements.write_text("\n".join(result)) return result @@ -743,7 +856,7 @@ def unpin_and_update_vendored(ctx, vendor=True, patched=False): def main(ctx, package=None): vendor_dir = _get_vendor_dir(ctx) patched_dir = _get_patched_dir(ctx) - log('Using vendor dir: %s' % vendor_dir) + log("Using vendor dir: %s" % vendor_dir) if package: vendor(ctx, vendor_dir, package=package) download_licenses(ctx, vendor_dir, package=package) @@ -752,13 +865,20 @@ def main(ctx, package=None): clean_vendor(ctx, vendor_dir) clean_vendor(ctx, patched_dir) vendor(ctx, vendor_dir) + install_pyyaml(ctx, patched_dir) vendor(ctx, patched_dir, rewrite=True) download_all_licenses(ctx, include_pip=True) # from .vendor_passa import vendor_passa # log("Vendoring passa...") # vendor_passa(ctx) # update_safety(ctx) - log('Revendoring complete') + log("Revendoring complete") + + +@invoke.task +def install_yaml(ctx): + patched_dir = _get_patched_dir(ctx) + install_pyyaml(ctx, patched_dir) @invoke.task diff --git a/tasks/vendoring/patches/patched/safety-main.patch b/tasks/vendoring/patches/patched/safety-main.patch new file mode 100644 index 00000000..3d214863 --- /dev/null +++ b/tasks/vendoring/patches/patched/safety-main.patch @@ -0,0 +1,57 @@ +diff --git a/pipenv/patched/safety/__main__.py b/pipenv/patched/safety/__main__.py +index d9a0bdab..f905408a 100644 +--- a/pipenv/patched/safety/__main__.py ++++ b/pipenv/patched/safety/__main__.py +@@ -1,8 +1,51 @@ + """Allow safety to be executable through `python -m safety`.""" + from __future__ import absolute_import + +-from .cli import cli ++import os ++import sys ++import sysconfig ++ ++ ++PATCHED_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ++PIPENV_DIR = os.path.dirname(PATCHED_DIR) ++VENDORED_DIR = os.path.join("PIPENV_DIR", "vendor") ++ ++ ++def get_site_packages(): ++ prefixes = {sys.prefix, sysconfig.get_config_var('prefix')} ++ try: ++ prefixes.add(sys.real_prefix) ++ except AttributeError: ++ pass ++ form = sysconfig.get_path('purelib', expand=False) ++ py_version_short = '{0[0]}.{0[1]}'.format(sys.version_info) ++ return { ++ form.format(base=prefix, py_version_short=py_version_short) ++ for prefix in prefixes ++ } ++ ++ ++def insert_before_site_packages(*paths): ++ site_packages = get_site_packages() ++ index = None ++ for i, path in enumerate(sys.path): ++ if path in site_packages: ++ index = i ++ break ++ if index is None: ++ sys.path += list(paths) ++ else: ++ sys.path = sys.path[:index] + list(paths) + sys.path[index:] ++ ++ ++def insert_pipenv_dirs(): ++ insert_before_site_packages(os.path.dirname(PIPENV_DIR), PATCHED_DIR, VENDORED_DIR) + + + if __name__ == "__main__": # pragma: no cover ++ insert_pipenv_dirs() ++ yaml_lib = "pipenv.patched.yaml{0}".format(sys.version_info[0]) ++ locals()[yaml_lib] = __import__(yaml_lib) ++ sys.modules["yaml"] = sys.modules[yaml_lib] ++ from safety.cli import cli + cli(prog_name="safety") diff --git a/tasks/vendoring/patches/vendor/dparse-configparser.patch b/tasks/vendoring/patches/vendor/dparse-configparser.patch new file mode 100644 index 00000000..bc61a75e --- /dev/null +++ b/tasks/vendoring/patches/vendor/dparse-configparser.patch @@ -0,0 +1,13 @@ +diff --git a/pipenv/vendor/dparse/parser.py b/pipenv/vendor/dparse/parser.py +index c01ebab4..9b2a0728 100644 +--- a/pipenv/vendor/dparse/parser.py ++++ b/pipenv/vendor/dparse/parser.py +@@ -6,7 +6,7 @@ import yaml + + from io import StringIO + +-from configparser import SafeConfigParser, NoOptionError ++from six.moves.configparser import SafeConfigParser, NoOptionError + + + from .regex import URL_REGEX, HASH_REGEX diff --git a/tasks/vendoring/safety/Pipfile b/tasks/vendoring/safety/Pipfile new file mode 100644 index 00000000..95769f0d --- /dev/null +++ b/tasks/vendoring/safety/Pipfile @@ -0,0 +1,9 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +safety = "*" diff --git a/tasks/vendoring/safety/__main__.py b/tasks/vendoring/safety/__main__.py new file mode 100644 index 00000000..9c6bbba2 --- /dev/null +++ b/tasks/vendoring/safety/__main__.py @@ -0,0 +1,45 @@ +"""Allow safety to be executable through `python -m safety`.""" +from __future__ import absolute_import + +import os +import sys +import sysconfig + +LIBPATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "lib") + + +def get_site_packages(): + prefixes = {sys.prefix, sysconfig.get_config_var('prefix')} + try: + prefixes.add(sys.real_prefix) + except AttributeError: + pass + form = sysconfig.get_path('purelib', expand=False) + py_version_short = '{0[0]}.{0[1]}'.format(sys.version_info) + return { + form.format(base=prefix, py_version_short=py_version_short) + for prefix in prefixes + } + + +def insert_before_site_packages(*paths): + site_packages = get_site_packages() + index = None + for i, path in enumerate(sys.path): + if path in site_packages: + index = i + break + if index is None: + sys.path += list(paths) + else: + sys.path = sys.path[:index] + list(paths) + sys.path[index:] + + + +if __name__ == "__main__": + insert_before_site_packages(LIBPATH) + yaml_lib = 'yaml{0}'.format(sys.version_info[0]) + locals()[yaml_lib] = __import__(yaml_lib) + sys.modules['yaml'] = sys.modules[yaml_lib] + from safety.cli import cli + cli(prog_name="safety") diff --git a/tasks/vendoring/safety/requirements.txt b/tasks/vendoring/safety/requirements.txt new file mode 100644 index 00000000..a521ea82 --- /dev/null +++ b/tasks/vendoring/safety/requirements.txt @@ -0,0 +1,29 @@ +pipenv==2018.11.26 +packaging==20.3 +safety==1.8.7 +pyyaml==5.3.1 +virtualenv==20.0.18 +requests==2.23.0 +idna==2.9 +pyparsing==2.4.7 +appdirs==1.4.3 +filelock==3.0.12 +distlib==0.3.0 +certifi==2020.4.5.1 +click==7.1.1 +chardet==3.0.4 +dparse==0.5.0 +virtualenv-clone==0.5.4 +urllib3==1.25.9 +six==1.14.0 +toml==0.10.0 +singledispatch==3.4.0.3 +configparser==4.0.2 +importlib-metadata==1.6.0 +typing==3.7.4.1 +pathlib2==2.3.5 +importlib-resources==1.4.0 +zipp==1.2.0 +contextlib2==0.6.0.post1 +enum34==1.1.10 +scandir==1.10.0 \ No newline at end of file diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b6141162..18e38878 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -129,6 +129,10 @@ def pytest_runtest_setup(item): sys.version_info >= (3, 7) ): pytest.skip('test only runs on python < 3.7') + if item.get_closest_marker('skip_py36') is not None and ( + sys.version_info[:2] == (3, 6) + ): + pytest.skip('test is skipped on python 3.6') @pytest.fixture diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index c7553952..84237eb5 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function import os +import sys import pytest @@ -130,7 +131,7 @@ funcsigs = "*" @pytest.mark.markers @pytest.mark.complex @pytest.mark.py3_only -@pytest.mark.lte_py36 +@pytest.mark.skip("test is flaky on CI") def test_resolver_unique_markers(PipenvInstance): """vcrpy has a dependency on `yarl` which comes with a marker of 'python version in "3.4, 3.5, 3.6" - this marker duplicates itself: