diff --git a/Pipfile.lock b/Pipfile.lock index 3990a451..30a618ac 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -626,10 +626,10 @@ }, "tomlkit": { "hashes": [ - "sha256:8ab16e93162fc44d3ad83d2aa29a7140b8f7d996ae1790a73b9a7aed6fb504ac", - "sha256:ca181cee7aee805d455628f7c94eb8ae814763769a93e69157f250fe4ebe1926" + "sha256:82a8fbb8d8c6af72e96ba00b9db3e20ef61be6c79082552c9363f4559702258b", + "sha256:a43e0195edc9b3c198cd4b5f0f3d427a395d47c4a76ceba7cc875ed030756c39" ], - "version": "==0.4.4" + "version": "==0.5.2" }, "towncrier": { "editable": true, diff --git a/pipenv/vendor/tomlkit/__init__.py b/pipenv/vendor/tomlkit/__init__.py index 89e4cf59..92bfa27c 100644 --- a/pipenv/vendor/tomlkit/__init__.py +++ b/pipenv/vendor/tomlkit/__init__.py @@ -22,4 +22,4 @@ from .api import value from .api import ws -__version__ = "0.4.6" +__version__ = "0.5.2" diff --git a/pipenv/vendor/tomlkit/_compat.py b/pipenv/vendor/tomlkit/_compat.py index f94bb10e..b7407af6 100644 --- a/pipenv/vendor/tomlkit/_compat.py +++ b/pipenv/vendor/tomlkit/_compat.py @@ -141,9 +141,11 @@ PY36 = sys.version_info >= (3, 6) if PY2: unicode = unicode chr = unichr + long = long else: unicode = str chr = chr + long = int def decode(string, encodings=None): diff --git a/pipenv/vendor/tomlkit/_utils.py b/pipenv/vendor/tomlkit/_utils.py index f62a354a..0a68be9f 100644 --- a/pipenv/vendor/tomlkit/_utils.py +++ b/pipenv/vendor/tomlkit/_utils.py @@ -9,19 +9,30 @@ from datetime import timedelta from ._compat import decode from ._compat import timezone +RFC_3339_LOOSE = re.compile( + "^" + r"(([0-9]+)-(\d{2})-(\d{2}))?" # Date + "(" + "([T ])?" # Separator + r"(\d{2}):(\d{2}):(\d{2})(\.([0-9]+))?" # Time + r"((Z)|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone + ")?" + "$" +) + RFC_3339_DATETIME = re.compile( "^" "([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])" # Date "[T ]" # Separator - "([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?" # Time - "((Z)|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone + r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?" # Time + r"((Z)|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone "$" ) RFC_3339_DATE = re.compile("^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$") RFC_3339_TIME = re.compile( - "^([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?$" + r"^([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?$" ) _utc = timezone(timedelta(), "UTC") diff --git a/pipenv/vendor/tomlkit/api.py b/pipenv/vendor/tomlkit/api.py index 0ac26752..e541c20c 100644 --- a/pipenv/vendor/tomlkit/api.py +++ b/pipenv/vendor/tomlkit/api.py @@ -1,5 +1,7 @@ import datetime as _datetime +from typing import Tuple + from ._utils import parse_rfc3339 from .container import Container from .items import AoT diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index 987a0790..cb8af1d5 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -1,5 +1,13 @@ from __future__ import unicode_literals +from typing import Any +from typing import Dict +from typing import Generator +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union + from ._compat import decode from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey @@ -9,7 +17,6 @@ from .items import Item from .items import Key from .items import Null from .items import Table -from .items import Trivia from .items import Whitespace from .items import item as _item @@ -74,7 +81,7 @@ class Container(dict): return self.append(key, item) - def append(self, key, item): # type: (Union[Key, str], Item) -> Container + def append(self, key, item): # type: (Union[Key, str, None], Item) -> Container if not isinstance(key, Key) and key is not None: key = Key(key) @@ -99,7 +106,11 @@ class Container(dict): self.append(None, Whitespace("\n")) if key is not None and key in self: - current = self._body[self._map[key]][1] + current_idx = self._map[key] + if isinstance(current_idx, tuple): + current_idx = current_idx[0] + + current = self._body[current_idx][1] if isinstance(item, Table): if not isinstance(current, (Table, AoT)): raise KeyAlreadyPresent(key) @@ -121,7 +132,7 @@ class Container(dict): current.append(k, v) return self - else: + elif not item.is_super_table(): raise KeyAlreadyPresent(key) elif isinstance(item, AoT): if not isinstance(current, AoT): @@ -173,7 +184,23 @@ class Container(dict): else: return self._insert_at(0, key, item) - self._map[key] = len(self._body) + if key in self._map: + current_idx = self._map[key] + if isinstance(current_idx, tuple): + current_idx = current_idx[0] + + current = self._body[current_idx][1] + if key is not None and not isinstance(current, Table): + raise KeyAlreadyPresent(key) + + # Adding sub tables to a currently existing table + idx = self._map[key] + if not isinstance(idx, tuple): + idx = (idx,) + + self._map[key] = idx + (len(self._body),) + else: + self._map[key] = len(self._body) self._body.append((key, item)) @@ -190,12 +217,12 @@ class Container(dict): if idx is None: raise NonExistentKey(key) - old_data = self._body[idx][1] - trivia = getattr(old_data, "trivia", None) - if trivia and getattr(trivia, "comment", None): - self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) + if isinstance(idx, tuple): + for i in idx: + self._body[i] = (None, Null()) else: self._body[idx] = (None, Null()) + super(Container, self).__delitem__(key.key) return self @@ -224,7 +251,16 @@ class Container(dict): # Increment indices after the current index for k, v in self._map.items(): - if v > idx: + if isinstance(v, tuple): + new_indices = [] + for v_ in v: + if v_ > idx: + v_ = v_ + 1 + + new_indices.append(v_) + + self._map[k] = tuple(new_indices) + elif v > idx: self._map[k] = v + 1 self._map[other_key] = idx + 1 @@ -257,7 +293,16 @@ class Container(dict): # Increment indices after the current index for k, v in self._map.items(): - if v >= idx: + if isinstance(v, tuple): + new_indices = [] + for v_ in v: + if v_ >= idx: + v_ = v_ + 1 + + new_indices.append(v_) + + self._map[k] = tuple(new_indices) + elif v >= idx: self._map[k] = v + 1 self._map[key] = idx @@ -286,29 +331,7 @@ class Container(dict): s = "" for k, v in self._body: if k is not None: - if False: - key = k.as_string() - - for _k, _v in v.value.body: - if _k is None: - s += v.as_string() - elif isinstance(_v, Table): - s += v.as_string(prefix=key) - else: - _key = key - if prefix is not None: - _key = prefix + "." + _key - - s += "{}{}{}{}{}{}{}".format( - _v.trivia.indent, - _key + "." + decode(_k.as_string()), - _k.sep, - decode(_v.as_string()), - _v.trivia.comment_ws, - decode(_v.trivia.comment), - _v.trivia.trail, - ) - elif isinstance(v, Table): + if isinstance(v, Table): s += self._render_table(k, v) elif isinstance(v, AoT): s += self._render_aot(k, v) @@ -332,7 +355,12 @@ class Container(dict): if prefix is not None: _key = prefix + "." + _key - if not table.is_super_table(): + if not table.is_super_table() or ( + any( + not isinstance(v, (Table, AoT, Whitespace)) for _, v in table.value.body + ) + and not key.is_dotted() + ): open_, close = "[", "]" if table.is_aot_element(): open_, close = "[[", "]]" @@ -465,7 +493,7 @@ class Container(dict): return key in self._map - def __getitem__(self, key): # type: (Union[Key, str]) -> Item + def __getitem__(self, key): # type: (Union[Key, str]) -> Union[Item, Container] if not isinstance(key, Key): key = Key(key) @@ -473,6 +501,20 @@ class Container(dict): if idx is None: raise NonExistentKey(key) + if isinstance(idx, tuple): + container = Container(True) + + for i in idx: + item = self._body[i][1] + + if isinstance(item, Table): + for k, v in item.value.body: + container.append(k, v) + else: + container.append(key, item) + + return container + item = self._body[idx][1] return item.value @@ -503,11 +545,20 @@ class Container(dict): def _replace_at( self, idx, new_key, value - ): # type: (int, Union[Key, str], Item) -> None + ): # type: (Union[int, Tuple[int]], Union[Key, str], Item) -> None + if isinstance(idx, tuple): + for i in idx[1:]: + self._body[i] = (None, Null()) + + idx = idx[0] + k, v = self._body[idx] self._map[new_key] = self._map.pop(k) + if isinstance(self._map[new_key], tuple): + self._map[new_key] = self._map[new_key][0] + value = _item(value) # Copying trivia @@ -517,6 +568,10 @@ class Container(dict): value.trivia.comment = v.trivia.comment value.trivia.trail = v.trivia.trail + if isinstance(value, Table): + # Insert a cosmetic new line for tables + value.append(None, Whitespace("\n")) + self._body[idx] = (new_key, value) super(Container, self).__setitem__(new_key.key, value.value) diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py index 46ee938b..4fbc667b 100644 --- a/pipenv/vendor/tomlkit/exceptions.py +++ b/pipenv/vendor/tomlkit/exceptions.py @@ -1,3 +1,6 @@ +from typing import Optional + + class TOMLKitError(Exception): pass @@ -23,6 +26,14 @@ class ParseError(ValueError, TOMLKitError): "{} at line {} col {}".format(message, self._line, self._col) ) + @property + def line(self): + return self._line + + @property + def col(self): + return self._col + class MixedArrayTypesError(ParseError): """ @@ -35,6 +46,50 @@ class MixedArrayTypesError(ParseError): super(MixedArrayTypesError, self).__init__(line, col, message=message) +class InvalidNumberError(ParseError): + """ + A numeric field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid number" + + super(InvalidNumberError, self).__init__(line, col, message=message) + + +class InvalidDateTimeError(ParseError): + """ + A datetime field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid datetime" + + super(InvalidDateTimeError, self).__init__(line, col, message=message) + + +class InvalidDateError(ParseError): + """ + A date field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid date" + + super(InvalidDateError, self).__init__(line, col, message=message) + + +class InvalidTimeError(ParseError): + """ + A date field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid time" + + super(InvalidTimeError, self).__init__(line, col, message=message) + + class InvalidNumberOrDateError(ParseError): """ A numeric or date field was improperly specified. @@ -46,6 +101,17 @@ class InvalidNumberOrDateError(ParseError): super(InvalidNumberOrDateError, self).__init__(line, col, message=message) +class InvalidUnicodeValueError(ParseError): + """ + A unicode code was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid unicode value" + + super(InvalidUnicodeValueError, self).__init__(line, col, message=message) + + class UnexpectedCharError(ParseError): """ An unexpected character was found during parsing. @@ -106,7 +172,9 @@ class InternalParserError(ParseError): An error that indicates a bug in the parser. """ - def __init__(self, line, col, message=None): # type: (int, int) -> None + def __init__( + self, line, col, message=None + ): # type: (int, int, Optional[str]) -> None msg = "Internal parser error" if message: msg += " ({})".format(message) diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index c3c2d59f..375b5f02 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -6,14 +6,18 @@ import string from datetime import date from datetime import datetime from datetime import time -import sys -if sys.version_info >= (3, 4): - from enum import Enum -else: - from pipenv.vendor.backports.enum import Enum +from enum import Enum +from typing import Any +from typing import Dict +from typing import Generator +from typing import List +from typing import Optional +from typing import Union + from ._compat import PY2 from ._compat import decode +from ._compat import long from ._compat import unicode from ._utils import escape_string @@ -21,7 +25,6 @@ if PY2: from pipenv.vendor.backports.functools_lru_cache import lru_cache else: from functools import lru_cache -from toml.decoder import InlineTableDict def item(value, _parent=None): @@ -37,10 +40,7 @@ def item(value, _parent=None): elif isinstance(value, float): return Float(value, Trivia(), str(value)) elif isinstance(value, dict): - if isinstance(value, InlineTableDict): - val = InlineTable(Container(), Trivia()) - else: - val = Table(Container(), Trivia(), False) + val = Table(Container(), Trivia(), False) for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): val[k] = item(v, _parent=val) @@ -124,6 +124,24 @@ class StringType(Enum): }[self] +class BoolType(Enum): + TRUE = "true" + FALSE = "false" + + @lru_cache(maxsize=None) + def __bool__(self): + return {BoolType.TRUE: True, BoolType.FALSE: False}[self] + + if PY2: + __nonzero__ = __bool__ # for PY2 + + def __iter__(self): + return iter(self.value) + + def __len__(self): + return len(self.value) + + class Trivia: """ Trivia information (aka metadata). @@ -310,7 +328,7 @@ class Comment(Item): return "{}{}".format(self._trivia.indent, decode(self._trivia.comment)) -class Integer(int, Item): +class Integer(long, Item): """ An integer literal. """ @@ -449,10 +467,10 @@ class Bool(Item): A boolean literal. """ - def __init__(self, value, trivia): # type: (float, Trivia) -> None + def __init__(self, t, trivia): # type: (float, Trivia) -> None super(Bool, self).__init__(trivia) - self._value = value + self._value = bool(t) @property def discriminant(self): # type: () -> int @@ -747,10 +765,6 @@ class Table(Item, dict): def discriminant(self): # type: () -> int return 9 - @property - def value(self): # type: () -> tomlkit.container.Container - return self._value - def add(self, key, item=None): # type: (Union[Key, Item, str], Any) -> Item if item is None: if not isinstance(key, (Comment, Whitespace)): @@ -924,6 +938,8 @@ class InlineTable(Item, dict): if not isinstance(_item, (Whitespace, Comment)): if not _item.trivia.indent and len(self._value) > 0: _item.trivia.indent = " " + if _item.trivia.comment: + _item.trivia.comment = "" self._value.append(key, _item) @@ -1003,8 +1019,7 @@ class InlineTable(Item, dict): if key is not None: super(InlineTable, self).__setitem__(key, value) - - if hasattr(value, "trivia") and value.trivia.comment: + if value.trivia.comment: value.trivia.comment = "" m = re.match("(?s)^[^ ]*([ ]+).*$", self._trivia.indent) diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py index 7971d9a2..7b948331 100644 --- a/pipenv/vendor/tomlkit/parser.py +++ b/pipenv/vendor/tomlkit/parser.py @@ -1,24 +1,31 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import datetime -import itertools import re import string -from copy import copy +from typing import Any +from typing import Generator +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union -from ._compat import PY2 from ._compat import chr from ._compat import decode from ._utils import _escaped +from ._utils import RFC_3339_LOOSE from ._utils import parse_rfc3339 from .container import Container from .exceptions import EmptyKeyError from .exceptions import EmptyTableNameError from .exceptions import InternalParserError from .exceptions import InvalidCharInStringError -from .exceptions import InvalidNumberOrDateError +from .exceptions import InvalidDateTimeError +from .exceptions import InvalidDateError +from .exceptions import InvalidTimeError +from .exceptions import InvalidNumberError +from .exceptions import InvalidUnicodeValueError from .exceptions import MixedArrayTypesError from .exceptions import ParseError from .exceptions import UnexpectedCharError @@ -26,12 +33,14 @@ from .exceptions import UnexpectedEofError from .items import AoT from .items import Array from .items import Bool +from .items import BoolType from .items import Comment from .items import Date from .items import DateTime from .items import Float from .items import InlineTable from .items import Integer +from .items import Item from .items import Key from .items import KeyType from .items import Null @@ -41,6 +50,7 @@ from .items import Table from .items import Time from .items import Trivia from .items import Whitespace +from .source import Source from .toml_char import TOMLChar from .toml_document import TOMLDocument @@ -52,68 +62,69 @@ class Parser: def __init__(self, string): # type: (str) -> None # Input to parse - self._src = decode(string) # type: str - # Iterator used for getting characters from src. - self._chars = iter([(i, TOMLChar(c)) for i, c in enumerate(self._src)]) - # Current byte offset into src. - self._idx = 0 - # Current character - self._current = TOMLChar("") # type: TOMLChar - # Index into src between which and idx slices will be extracted - self._marker = 0 + self._src = Source(decode(string)) self._aot_stack = [] - self.inc() + @property + def _state(self): + return self._src.state + + @property + def _idx(self): + return self._src.idx + + @property + def _current(self): + return self._src.current + + @property + def _marker(self): + return self._src.marker def extract(self): # type: () -> str """ Extracts the value between marker and index """ - if self.end(): - return self._src[self._marker :] - else: - return self._src[self._marker : self._idx] + return self._src.extract() - def inc(self, exception=None): # type: () -> bool + def inc(self, exception=None): # type: (Optional[ParseError.__class__]) -> bool """ Increments the parser if the end of the input has not been reached. Returns whether or not it was able to advance. """ - try: - self._idx, self._current = next(self._chars) + return self._src.inc(exception=exception) - return True - except StopIteration: - self._idx = len(self._src) - self._current = TOMLChar("\0") - - if not exception: - return False - raise exception - - def inc_n(self, n, exception=None): # type: (int) -> bool + def inc_n(self, n, exception=None): # type: (int, Optional[ParseError]) -> bool """ Increments the parser by n characters if the end of the input has not been reached. """ - for _ in range(n): - if not self.inc(exception=exception): - return False + return self._src.inc_n(n=n, exception=exception) - return True + def consume(self, chars, min=0, max=-1): + """ + Consume chars until min/max is satisfied is valid. + """ + return self._src.consume(chars=chars, min=min, max=max) def end(self): # type: () -> bool """ Returns True if the parser has reached the end of the input. """ - return self._idx >= len(self._src) or self._current == "\0" + return self._src.end() def mark(self): # type: () -> None """ Sets the marker to the index's current position """ - self._marker = self._idx + self._src.mark() + + def parse_error(self, exception=ParseError, *args): + """ + Creates a generic "parse error" at the current position. + """ + return self._src.parse_error(exception, *args) def parse(self): # type: () -> TOMLDocument body = TOMLDocument(True) @@ -173,27 +184,6 @@ class Parser: return True - def parse_error(self, kind=ParseError, args=None): # type: () -> None - """ - Creates a generic "parse error" at the current position. - """ - line, col = self._to_linecol(self._idx) - - if args: - return kind(line, col, *args) - else: - return kind(line, col) - - def _to_linecol(self, offset): # type: (int) -> Tuple[int, int] - cur = 0 - for i, line in enumerate(self._src.splitlines()): - if cur + len(line) + 1 > offset: - return (i + 1, offset - cur) - - cur += len(line) + 1 - - return len(self._src.splitlines()), 0 - def _is_child(self, parent, child): # type: (str, str) -> bool """ Returns whether a key is strictly a child of another key. @@ -256,55 +246,35 @@ class Parser: if the item is value-like. """ self.mark() - saved_idx = self._save_idx() + with self._state as state: + while True: + c = self._current + if c == "\n": + # Found a newline; Return all whitespace found up to this point. + self.inc() - while True: - c = self._current - if c == "\n": - # Found a newline; Return all whitespace found up to this point. - self.inc() + return None, Whitespace(self.extract()) + elif c in " \t\r": + # Skip whitespace. + if not self.inc(): + return None, Whitespace(self.extract()) + elif c == "#": + # Found a comment, parse it + indent = self.extract() + cws, comment, trail = self._parse_comment_trail() - return (None, Whitespace(self.extract())) - elif c in " \t\r": - # Skip whitespace. - if not self.inc(): - return (None, Whitespace(self.extract())) - elif c == "#": - # Found a comment, parse it - indent = self.extract() - cws, comment, trail = self._parse_comment_trail() + return None, Comment(Trivia(indent, cws, comment, trail)) + elif c == "[": + # Found a table, delegate to the calling function. + return + else: + # Begining of a KV pair. + # Return to beginning of whitespace so it gets included + # as indentation for the KV about to be parsed. + state.restore = True + break - return (None, Comment(Trivia(indent, cws, comment, trail))) - elif c == "[": - # Found a table, delegate to the calling function. - return - else: - # Begining of a KV pair. - # Return to beginning of whitespace so it gets included - # as indentation for the KV about to be parsed. - self._restore_idx(*saved_idx) - key, value = self._parse_key_value(True) - - return key, value - - def _save_idx(self): # type: () -> Tuple[Iterator, int, str] - if PY2: - # Python 2.7 does not allow to directly copy - # an iterator, so we have to make tees of the original - # chars iterator. - chars1, chars2 = itertools.tee(self._chars) - - # We can no longer use the original chars iterator. - self._chars = chars1 - - return chars2, self._idx, self._current - - return copy(self._chars), self._idx, self._current - - def _restore_idx(self, chars, idx, current): # type: (Iterator, int, str) -> None - self._chars = chars - self._idx = idx - self._current = current + return self._parse_key_value(True) def _parse_comment_trail(self): # type: () -> Tuple[str, str, str] """ @@ -341,7 +311,7 @@ class Parser: elif c in " \t\r": self.inc() else: - raise self.parse_error(UnexpectedCharError, (c)) + raise self.parse_error(UnexpectedCharError, c) if self.end(): break @@ -361,9 +331,7 @@ class Parser: return comment_ws, comment, trail - def _parse_key_value( - self, parse_comment=False, inline=True - ): # type: (bool, bool) -> (Key, Item) + def _parse_key_value(self, parse_comment=False): # type: (bool) -> (Key, Item) # Leading indent self.mark() @@ -383,7 +351,7 @@ class Parser: while self._current.is_kv_sep() and self.inc(): if self._current == "=": if found_equals: - raise self.parse_error(UnexpectedCharError, ("=",)) + raise self.parse_error(UnexpectedCharError, "=") else: found_equals = True pass @@ -473,7 +441,7 @@ class Parser: def _handle_dotted_key( self, container, key, value - ): # type: (Container, Key) -> None + ): # type: (Container, Key, Any) -> None names = tuple(self._split_table_name(key.key)) name = names[0] name._dotted = True @@ -510,84 +478,22 @@ class Parser: Attempts to parse a value at the current position. """ self.mark() + c = self._current trivia = Trivia() - c = self._current - if c == '"': + if c == StringType.SLB.value: return self._parse_basic_string() - elif c == "'": + elif c == StringType.SLL.value: return self._parse_literal_string() - elif c == "t" and self._src[self._idx :].startswith("true"): - # Boolean: true - self.inc_n(4) - - return Bool(True, trivia) - elif c == "f" and self._src[self._idx :].startswith("false"): - # Boolean: true - self.inc_n(5) - - return Bool(False, trivia) + elif c == BoolType.TRUE.value[0]: + return self._parse_true() + elif c == BoolType.FALSE.value[0]: + return self._parse_false() elif c == "[": - # Array - elems = [] # type: List[Item] - self.inc() - - while self._current != "]": - self.mark() - while self._current.is_ws() or self._current == ",": - self.inc() - - if self._idx != self._marker: - elems.append(Whitespace(self.extract())) - - if self._current == "]": - break - - if self._current == "#": - cws, comment, trail = self._parse_comment_trail() - - next_ = Comment(Trivia("", cws, comment, trail)) - else: - next_ = self._parse_value() - - elems.append(next_) - - self.inc() - - try: - res = Array(elems, trivia) - except ValueError: - raise self.parse_error(MixedArrayTypesError) - - if res.is_homogeneous(): - return res - - raise self.parse_error(MixedArrayTypesError) + return self._parse_array() elif c == "{": - # Inline table - elems = Container(True) - self.inc() - - while self._current != "}": - self.mark() - while self._current.is_spaces() or self._current == ",": - self.inc() - - if self._idx != self._marker: - ws = self.extract().lstrip(",") - if ws: - elems.append(None, Whitespace(ws)) - - if self._current == "}": - break - - key, val = self._parse_key_value(False, inline=True) - elems.append(key, val) - - self.inc() - - return InlineTable(elems, trivia) - elif c in string.digits + "+-" or self._peek(4) in { + return self._parse_inline_table() + elif c in "+-" or self._peek(4) in { "+inf", "-inf", "inf", @@ -595,7 +501,7 @@ class Parser: "-nan", "nan", }: - # Integer, Float, Date, Time or DateTime + # Number while self._current not in " \t\n\r#,]}" and self.inc(): pass @@ -605,24 +511,166 @@ class Parser: if item is not None: return item - try: - res = parse_rfc3339(raw) - except ValueError: - res = None + raise self.parse_error(InvalidNumberError) + elif c in string.digits: + # Integer, Float, Date, Time or DateTime + while self._current not in " \t\n\r#,]}" and self.inc(): + pass - if res is None: - raise self.parse_error(InvalidNumberOrDateError) + raw = self.extract() - if isinstance(res, datetime.datetime): - return DateTime(res, trivia, raw) - elif isinstance(res, datetime.time): - return Time(res, trivia, raw) - elif isinstance(res, datetime.date): - return Date(res, trivia, raw) - else: - raise self.parse_error(InvalidNumberOrDateError) + m = RFC_3339_LOOSE.match(raw) + if m: + if m.group(1) and m.group(5): + # datetime + try: + return DateTime(parse_rfc3339(raw), trivia, raw) + except ValueError: + raise self.parse_error(InvalidDateTimeError) + + if m.group(1): + try: + return Date(parse_rfc3339(raw), trivia, raw) + except ValueError: + raise self.parse_error(InvalidDateError) + + if m.group(5): + try: + return Time(parse_rfc3339(raw), trivia, raw) + except ValueError: + raise self.parse_error(InvalidTimeError) + + item = self._parse_number(raw, trivia) + if item is not None: + return item + + raise self.parse_error(InvalidNumberError) else: - raise self.parse_error(UnexpectedCharError, (c)) + raise self.parse_error(UnexpectedCharError, c) + + def _parse_true(self): + return self._parse_bool(BoolType.TRUE) + + def _parse_false(self): + return self._parse_bool(BoolType.FALSE) + + def _parse_bool(self, style): # type: (BoolType) -> Bool + with self._state: + style = BoolType(style) + + # only keep parsing for bool if the characters match the style + # try consuming rest of chars in style + for c in style: + self.consume(c, min=1, max=1) + + return Bool(style, Trivia()) + + def _parse_array(self): # type: () -> Array + # Consume opening bracket, EOF here is an issue (middle of array) + self.inc(exception=UnexpectedEofError) + + elems = [] # type: List[Item] + prev_value = None + while True: + # consume whitespace + mark = self._idx + self.consume(TOMLChar.SPACES) + newline = self.consume(TOMLChar.NL) + indent = self._src[mark : self._idx] + if newline: + elems.append(Whitespace(indent)) + continue + + # consume comment + if self._current == "#": + cws, comment, trail = self._parse_comment_trail() + elems.append(Comment(Trivia(indent, cws, comment, trail))) + continue + + # consume indent + if indent: + elems.append(Whitespace(indent)) + continue + + # consume value + if not prev_value: + try: + elems.append(self._parse_value()) + prev_value = True + continue + except UnexpectedCharError: + pass + + # consume comma + if prev_value and self._current == ",": + self.inc(exception=UnexpectedEofError) + elems.append(Whitespace(",")) + prev_value = False + continue + + # consume closing bracket + if self._current == "]": + # consume closing bracket, EOF here doesn't matter + self.inc() + break + + raise self.parse_error(UnexpectedCharError, self._current) + + try: + res = Array(elems, Trivia()) + except ValueError: + pass + else: + if res.is_homogeneous(): + return res + + raise self.parse_error(MixedArrayTypesError) + + def _parse_inline_table(self): # type: () -> InlineTable + # consume opening bracket, EOF here is an issue (middle of array) + self.inc(exception=UnexpectedEofError) + + elems = Container(True) + trailing_comma = None + while True: + # consume leading whitespace + mark = self._idx + self.consume(TOMLChar.SPACES) + raw = self._src[mark : self._idx] + if raw: + elems.add(Whitespace(raw)) + + if not trailing_comma: + # None: empty inline table + # False: previous key-value pair was not followed by a comma + if self._current == "}": + # consume closing bracket, EOF here doesn't matter + self.inc() + break + if trailing_comma is False: + raise self.parse_error(UnexpectedCharError, self._current) + else: + # True: previous key-value pair was followed by a comma + if self._current == "}": + raise self.parse_error(UnexpectedCharError, self._current) + + key, val = self._parse_key_value(False) + elems.add(key, val) + + # consume trailing whitespace + mark = self._idx + self.consume(TOMLChar.SPACES) + raw = self._src[mark : self._idx] + if raw: + elems.add(Whitespace(raw)) + + # consume trailing comma + trailing_comma = self._current == "," + if trailing_comma: + # consume closing bracket, EOF here is an issue (middle of inline table) + self.inc(exception=UnexpectedEofError) + + return InlineTable(elems, Trivia()) def _parse_number(self, raw, trivia): # type: (str, Trivia) -> Optional[Item] # Leading zeros are not allowed @@ -670,11 +718,13 @@ class Parser: except ValueError: return - def _parse_literal_string(self): # type: () -> Item - return self._parse_string(StringType.SLL) + def _parse_literal_string(self): # type: () -> String + with self._state: + return self._parse_string(StringType.SLL) - def _parse_basic_string(self): # type: () -> Item - return self._parse_string(StringType.SLB) + def _parse_basic_string(self): # type: () -> String + with self._state: + return self._parse_string(StringType.SLB) def _parse_escaped_char(self, multiline): if multiline and self._current.is_ws(): @@ -696,7 +746,7 @@ class Parser: # the escape followed by whitespace must have a newline # before any other chars if "\n" not in tmp: - raise self.parse_error(InvalidCharInStringError, (self._current,)) + raise self.parse_error(InvalidCharInStringError, self._current) return "" @@ -717,15 +767,17 @@ class Parser: return u - raise self.parse_error(InvalidCharInStringError, (self._current,)) + raise self.parse_error(InvalidUnicodeValueError) - def _parse_string(self, delim): # type: (str) -> Item - delim = StringType(delim) - assert delim.is_singleline() + raise self.parse_error(InvalidCharInStringError, self._current) + def _parse_string(self, delim): # type: (StringType) -> String # only keep parsing for string if the current character matches the delim if self._current != delim.unit: - raise ValueError("Expecting a {!r} character".format(delim)) + raise self.parse_error( + InternalParserError, + "Invalid character for string type {}".format(delim), + ) # consume the opening/first delim, EOF here is an issue # (middle of string or middle of delim) @@ -755,7 +807,7 @@ class Parser: while True: if delim.is_singleline() and self._current.is_nl(): # single line cannot have actual newline characters - raise self.parse_error(InvalidCharInStringError, (self._current,)) + raise self.parse_error(InvalidCharInStringError, self._current) elif not escaped and self._current == delim.unit: # try to process current as a closing delim original = self.extract() @@ -781,8 +833,6 @@ class Parser: if not close: # if there is no close characters, keep parsing continue else: - close = delim.unit - # consume the closing delim, we do not care if EOF occurs as # that would simply imply the end of self._src self.inc() @@ -817,8 +867,7 @@ class Parser: """ if self._current != "[": raise self.parse_error( - InternalParserError, - ("_parse_table() called on non-bracket character.",), + InternalParserError, "_parse_table() called on non-bracket character." ) indent = self.extract() @@ -945,7 +994,7 @@ class Parser: else: raise self.parse_error( InternalParserError, - ("_parse_item() returned None on a non-bracket character.",), + "_parse_item() returned None on a non-bracket character.", ) if isinstance(result, Null): @@ -970,32 +1019,27 @@ class Parser: Returns the name of the table about to be parsed, as well as whether it is part of an AoT. """ - # Save initial state - idx = self._save_idx() - marker = self._marker + # we always want to restore after exiting this scope + with self._state(save_marker=True, restore=True): + if self._current != "[": + raise self.parse_error( + InternalParserError, + "_peek_table() entered on non-bracket character", + ) - if self._current != "[": - raise self.parse_error( - InternalParserError, ("_peek_table() entered on non-bracket character",) - ) - - # AoT - self.inc() - is_aot = False - if self._current == "[": + # AoT self.inc() - is_aot = True + is_aot = False + if self._current == "[": + self.inc() + is_aot = True - self.mark() + self.mark() - while self._current != "]" and self.inc(): - table_name = self.extract() + while self._current != "]" and self.inc(): + table_name = self.extract() - # Restore initial state - self._restore_idx(*idx) - self._marker = marker - - return is_aot, table_name + return is_aot, table_name def _parse_aot(self, first, name_first): # type: (Table, str) -> AoT """ @@ -1022,57 +1066,53 @@ class Parser: n is the max number of characters that will be peeked. """ - idx = self._save_idx() - buf = "" - for _ in range(n): - if self._current not in " \t\n\r#,]}": - buf += self._current - self.inc() - continue + # we always want to restore after exiting this scope + with self._state(restore=True): + buf = "" + for _ in range(n): + if self._current not in " \t\n\r#,]}": + buf += self._current + self.inc() + continue - break + break + return buf - self._restore_idx(*idx) - - return buf - - def _peek_unicode(self, is_long): # type: () -> Tuple[bool, str] + def _peek_unicode( + self, is_long + ): # type: (bool) -> Tuple[Optional[str], Optional[str]] """ Peeks ahead non-intrusively by cloning then restoring the initial state of the parser. Returns the unicode value is it's a valid one else None. """ - # Save initial state - idx = self._save_idx() - marker = self._marker + # we always want to restore after exiting this scope + with self._state(save_marker=True, restore=True): + if self._current not in {"u", "U"}: + raise self.parse_error( + InternalParserError, "_peek_unicode() entered on non-unicode value" + ) - if self._current not in {"u", "U"}: - raise self.parse_error( - InternalParserError, ("_peek_unicode() entered on non-unicode value") - ) + self.inc() # Dropping prefix + self.mark() - # AoT - self.inc() # Dropping prefix - self.mark() + if is_long: + chars = 8 + else: + chars = 4 - if is_long: - chars = 8 - else: - chars = 4 + if not self.inc_n(chars): + value, extracted = None, None + else: + extracted = self.extract() - if not self.inc_n(chars): - value, extracted = None, None - else: - extracted = self.extract() + if extracted[0].lower() == "d" and extracted[1].strip("01234567"): + return None, None - try: - value = chr(int(extracted, 16)) - except ValueError: - value = None + try: + value = chr(int(extracted, 16)) + except ValueError: + value = None - # Restore initial state - self._restore_idx(*idx) - self._marker = marker - - return value, extracted + return value, extracted diff --git a/pipenv/vendor/tomlkit/source.py b/pipenv/vendor/tomlkit/source.py new file mode 100644 index 00000000..1a96e058 --- /dev/null +++ b/pipenv/vendor/tomlkit/source.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import itertools + +from copy import copy +from typing import Optional +from typing import Tuple + +from ._compat import PY2 +from ._compat import unicode +from .exceptions import UnexpectedEofError +from .exceptions import UnexpectedCharError +from .exceptions import ParseError +from .toml_char import TOMLChar + + +class _State: + def __init__( + self, source, save_marker=False, restore=False + ): # type: (_Source, Optional[bool], Optional[bool]) -> None + self._source = source + self._save_marker = save_marker + self.restore = restore + + def __enter__(self): # type: () -> None + # Entering this context manager - save the state + if PY2: + # Python 2.7 does not allow to directly copy + # an iterator, so we have to make tees of the original + # chars iterator. + self._source._chars, self._chars = itertools.tee(self._source._chars) + else: + self._chars = copy(self._source._chars) + self._idx = self._source._idx + self._current = self._source._current + self._marker = self._source._marker + + return self + + def __exit__(self, exception_type, exception_val, trace): + # Exiting this context manager - restore the prior state + if self.restore or exception_type: + self._source._chars = self._chars + self._source._idx = self._idx + self._source._current = self._current + if self._save_marker: + self._source._marker = self._marker + + # Restore exceptions are silently consumed, other exceptions need to + # propagate + return exception_type is None + + +class _StateHandler: + """ + State preserver for the Parser. + """ + + def __init__(self, source): # type: (Source) -> None + self._source = source + self._states = [] + + def __call__(self, *args, **kwargs): + return _State(self._source, *args, **kwargs) + + def __enter__(self): # type: () -> None + state = self() + self._states.append(state) + return state.__enter__() + + def __exit__(self, exception_type, exception_val, trace): + state = self._states.pop() + return state.__exit__(exception_type, exception_val, trace) + + +class Source(unicode): + EOF = TOMLChar("\0") + + def __init__(self, _): # type: (unicode) -> None + super(Source, self).__init__() + + # Collection of TOMLChars + self._chars = iter([(i, TOMLChar(c)) for i, c in enumerate(self)]) + + self._idx = 0 + self._marker = 0 + self._current = TOMLChar("") + + self._state = _StateHandler(self) + + self.inc() + + def reset(self): + # initialize both idx and current + self.inc() + + # reset marker + self.mark() + + @property + def state(self): # type: () -> _StateHandler + return self._state + + @property + def idx(self): # type: () -> int + return self._idx + + @property + def current(self): # type: () -> TOMLChar + return self._current + + @property + def marker(self): # type: () -> int + return self._marker + + def extract(self): # type: () -> unicode + """ + Extracts the value between marker and index + """ + return self[self._marker : self._idx] + + def inc(self, exception=None): # type: (Optional[ParseError.__class__]) -> bool + """ + Increments the parser if the end of the input has not been reached. + Returns whether or not it was able to advance. + """ + try: + self._idx, self._current = next(self._chars) + + return True + except StopIteration: + self._idx = len(self) + self._current = self.EOF + if exception: + raise self.parse_error(exception) + + return False + + def inc_n(self, n, exception=None): # type: (int, Exception) -> bool + """ + Increments the parser by n characters + if the end of the input has not been reached. + """ + for _ in range(n): + if not self.inc(exception=exception): + return False + + return True + + def consume(self, chars, min=0, max=-1): + """ + Consume chars until min/max is satisfied is valid. + """ + while self.current in chars and max != 0: + min -= 1 + max -= 1 + if not self.inc(): + break + + # failed to consume minimum number of characters + if min > 0: + self.parse_error(UnexpectedCharError) + + def end(self): # type: () -> bool + """ + Returns True if the parser has reached the end of the input. + """ + return self._current is self.EOF + + def mark(self): # type: () -> None + """ + Sets the marker to the index's current position + """ + self._marker = self._idx + + def parse_error( + self, exception=ParseError, *args + ): # type: (ParseError.__class__, ...) -> ParseError + """ + Creates a generic "parse error" at the current position. + """ + line, col = self._to_linecol() + + return exception(line, col, *args) + + def _to_linecol(self): # type: () -> Tuple[int, int] + cur = 0 + for i, line in enumerate(self.splitlines()): + if cur + len(line) + 1 > self.idx: + return (i + 1, self.idx - cur) + + cur += len(line) + 1 + + return len(self.splitlines()), 0 diff --git a/pipenv/vendor/tomlkit/toml_char.py b/pipenv/vendor/tomlkit/toml_char.py index 5164ea8b..02c55172 100644 --- a/pipenv/vendor/tomlkit/toml_char.py +++ b/pipenv/vendor/tomlkit/toml_char.py @@ -16,44 +16,51 @@ class TOMLChar(unicode): if len(self) > 1: raise ValueError("A TOML character must be of length 1") + BARE = string.ascii_letters + string.digits + "-_" + KV = "= \t" + NUMBER = string.digits + "+-_.e" + SPACES = " \t" + NL = "\n\r" + WS = SPACES + NL + @lru_cache(maxsize=None) def is_bare_key_char(self): # type: () -> bool """ Whether the character is a valid bare key name or not. """ - return self in string.ascii_letters + string.digits + "-" + "_" + return self in self.BARE @lru_cache(maxsize=None) def is_kv_sep(self): # type: () -> bool """ Whether the character is a valid key/value separator ot not. """ - return self in "= \t" + return self in self.KV @lru_cache(maxsize=None) def is_int_float_char(self): # type: () -> bool """ Whether the character if a valid integer or float value character or not. """ - return self in string.digits + "+" + "-" + "_" + "." + "e" + return self in self.NUMBER @lru_cache(maxsize=None) def is_ws(self): # type: () -> bool """ Whether the character is a whitespace character or not. """ - return self in " \t\r\n" + return self in self.WS @lru_cache(maxsize=None) def is_nl(self): # type: () -> bool """ Whether the character is a new line character or not. """ - return self in "\n\r" + return self in self.NL @lru_cache(maxsize=None) def is_spaces(self): # type: () -> bool """ Whether the character is a space or not """ - return self in " \t" + return self in self.SPACES diff --git a/pipenv/vendor/tomlkit/toml_file.py b/pipenv/vendor/tomlkit/toml_file.py index 631e9959..3b416664 100644 --- a/pipenv/vendor/tomlkit/toml_file.py +++ b/pipenv/vendor/tomlkit/toml_file.py @@ -1,5 +1,8 @@ import io +from typing import Any +from typing import Dict + from .api import loads from .toml_document import TOMLDocument diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 45ff0384..3228068f 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -51,3 +51,4 @@ git+https://github.com/sarugaku/passa.git@master#egg=passa cursor==1.2.0 resolvelib==0.2.2 backports.functools_lru_cache==1.5 +tomlkit \ No newline at end of file