diff --git a/news/4433.bugfix.rst b/news/4433.bugfix.rst new file mode 100644 index 00000000..77b728f6 --- /dev/null +++ b/news/4433.bugfix.rst @@ -0,0 +1 @@ +Fix a bug that compound TOML table is not parsed correctly. diff --git a/news/4433.vendor.rst b/news/4433.vendor.rst new file mode 100644 index 00000000..786042e3 --- /dev/null +++ b/news/4433.vendor.rst @@ -0,0 +1 @@ +Update ``tomlkit`` from ``0.5.11`` to ``0.7.0``. diff --git a/pipenv/vendor/tomlkit/__init__.py b/pipenv/vendor/tomlkit/__init__.py index ab126006..e0a7a542 100644 --- a/pipenv/vendor/tomlkit/__init__.py +++ b/pipenv/vendor/tomlkit/__init__.py @@ -22,4 +22,4 @@ from .api import value from .api import ws -__version__ = "0.5.11" +__version__ = "0.7.0" diff --git a/pipenv/vendor/tomlkit/_compat.py b/pipenv/vendor/tomlkit/_compat.py index 768ce338..8d3b0ae3 100644 --- a/pipenv/vendor/tomlkit/_compat.py +++ b/pipenv/vendor/tomlkit/_compat.py @@ -1,6 +1,7 @@ import re import sys + try: from datetime import timezone except ImportError: @@ -149,6 +150,12 @@ else: long = int +if PY36: + OrderedDict = dict +else: + from collections import OrderedDict + + def decode(string, encodings=None): if not PY2 and not isinstance(string, bytes): return string diff --git a/pipenv/vendor/tomlkit/_utils.py b/pipenv/vendor/tomlkit/_utils.py index 0a68be9f..2ae3e424 100644 --- a/pipenv/vendor/tomlkit/_utils.py +++ b/pipenv/vendor/tomlkit/_utils.py @@ -4,11 +4,18 @@ from datetime import date from datetime import datetime from datetime import time from datetime import timedelta - +from typing import Union from ._compat import decode from ._compat import timezone + +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping + + RFC_3339_LOOSE = re.compile( "^" r"(([0-9]+)-(\d{2})-(\d{2}))?" # Date @@ -52,8 +59,6 @@ def parse_rfc3339(string): # type: (str) -> Union[datetime, date, time] if m.group(7): microsecond = int(("{:<06s}".format(m.group(8)))[:6]) - dt = datetime(year, month, day, hour, minute, second, microsecond) - if m.group(9): # Timezone tz = m.group(9) @@ -129,3 +134,11 @@ def escape_string(s): flush() return "".join(res) + + +def merge_dicts(d1, d2): + for k, v in d2.items(): + if k in d1 and isinstance(d1[k], dict) and isinstance(d2[k], Mapping): + merge_dicts(d1[k], d2[k]) + else: + d1[k] = d2[k] diff --git a/pipenv/vendor/tomlkit/api.py b/pipenv/vendor/tomlkit/api.py index c9621830..3de41219 100644 --- a/pipenv/vendor/tomlkit/api.py +++ b/pipenv/vendor/tomlkit/api.py @@ -1,26 +1,28 @@ import datetime as _datetime +from typing import Tuple + from ._utils import parse_rfc3339 from .container import Container from .items import AoT -from .items import Comment -from .items import InlineTable -from .items import Item as _Item from .items import Array from .items import Bool -from .items import Key +from .items import Comment from .items import Date from .items import DateTime from .items import Float -from .items import Table +from .items import InlineTable from .items import Integer +from .items import Item as _Item +from .items import Key +from .items import String +from .items import Table +from .items import Time from .items import Trivia from .items import Whitespace -from .items import String from .items import item from .parser import Parser from .toml_document import TOMLDocument as _TOMLDocument -from .items import Time def loads(string): # type: (str) -> _TOMLDocument @@ -32,12 +34,12 @@ def loads(string): # type: (str) -> _TOMLDocument return parse(string) -def dumps(data): # type: (_TOMLDocument) -> str +def dumps(data, sort_keys=False): # type: (_TOMLDocument, bool) -> str """ Dumps a TOMLDocument into a string. """ if not isinstance(data, _TOMLDocument) and isinstance(data, dict): - data = item(data) + data = item(data, _sort_keys=sort_keys) return data.as_string() diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index 96415901..6386e738 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -2,16 +2,26 @@ from __future__ import unicode_literals import copy +from typing import Any +from typing import Dict +from typing import Generator +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union + from ._compat import decode +from ._utils import merge_dicts from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey +from .exceptions import ParseError +from .exceptions import TOMLKitError from .items import AoT from .items import Comment from .items import Item from .items import Key from .items import Null from .items import Table -from .items import Trivia from .items import Whitespace from .items import item as _item @@ -28,6 +38,7 @@ class Container(dict): self._map = {} # type: Dict[Key, int] self._body = [] # type: List[Tuple[Optional[Key], Item]] self._parsed = parsed + self._table_keys = [] @property def body(self): # type: () -> List[Tuple[Optional[Key], Item]] @@ -47,7 +58,7 @@ class Container(dict): v = v.value if k in d: - d[k].update(v) + merge_dicts(d[k], v) else: d[k] = v @@ -106,9 +117,12 @@ class Container(dict): if key is not None and key in self: current_idx = self._map[key] if isinstance(current_idx, tuple): - current_idx = current_idx[0] + current_body_element = self._body[current_idx[-1]] + else: + current_body_element = self._body[current_idx] + + current = current_body_element[1] - current = self._body[current_idx][1] if isinstance(item, Table): if not isinstance(current, (Table, AoT)): raise KeyAlreadyPresent(key) @@ -123,17 +137,46 @@ class Container(dict): else: current.append(item) + return self + elif current.is_aot(): + if not item.is_aot_element(): + # Tried to define a table after an AoT with the same name. + raise KeyAlreadyPresent(key) + + current.append(item) + return self elif current.is_super_table(): if item.is_super_table(): + # We need to merge both super tables + if ( + self._table_keys[-1] != current_body_element[0] + or key.is_dotted() + or current_body_element[0].is_dotted() + ): + if not isinstance(current_idx, tuple): + current_idx = (current_idx,) + + self._map[key] = current_idx + (len(self._body),) + self._body.append((key, item)) + self._table_keys.append(key) + + # Building a temporary proxy to check for errors + OutOfOrderTableProxy(self, self._map[key]) + + return self + for k, v in item.value.body: current.append(k, v) return self + elif current_body_element[0].is_dotted(): + raise TOMLKitError("Redefinition of an existing table") elif not item.is_super_table(): raise KeyAlreadyPresent(key) elif isinstance(item, AoT): if not isinstance(current, AoT): + # Tried to define an AoT after a table with the same name. raise KeyAlreadyPresent(key) for table in item.body: @@ -185,22 +228,23 @@ class Container(dict): if key in self._map: current_idx = self._map[key] if isinstance(current_idx, tuple): - current_idx = current_idx[0] + current_idx = current_idx[-1] current = self._body[current_idx][1] if key is not None and not isinstance(current, Table): raise KeyAlreadyPresent(key) # Adding sub tables to a currently existing table - idx = self._map[key] - if not isinstance(idx, tuple): - idx = (idx,) + if not isinstance(current_idx, tuple): + current_idx = (current_idx,) - self._map[key] = idx + (len(self._body),) + self._map[key] = current_idx + (len(self._body),) else: self._map[key] = len(self._body) self._body.append((key, item)) + if item.is_table(): + self._table_keys.append(key) if key is not None: super(Container, self).__setitem__(key.key, item.value) @@ -219,12 +263,7 @@ class Container(dict): for i in idx: self._body[i] = (None, Null()) else: - old_data = self._body[idx][1] - trivia = getattr(old_data, "trivia", None) - if trivia and trivia.comment: - self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) - else: - self._body[idx] = (None, Null()) + self._body[idx] = (None, Null()) super(Container, self).__delitem__(key.key) @@ -327,13 +366,19 @@ class Container(dict): if idx is None: raise NonExistentKey(key) + if isinstance(idx, tuple): + # The item we are getting is an out of order table + # so we need a proxy to retrieve the proper objects + # from the parent container + return OutOfOrderTableProxy(self, idx) + return self._body[idx][1] def last_item(self): # type: () -> Optional[Item] if self._body: return self._body[-1][1] - def as_string(self, prefix=None): # type: () -> str + def as_string(self): # type: () -> str s = "" for k, v in self._body: if k is not None: @@ -469,18 +514,11 @@ class Container(dict): # Dictionary methods def keys(self): # type: () -> Generator[str] - for k, _ in self._body: - if k is None: - continue - - yield k.key + return super(Container, self).keys() def values(self): # type: () -> Generator[Item] - for k, v in self._body: - if k is None: - continue - - yield v.value + for k in self.keys(): + yield self[k] def items(self): # type: () -> Generator[Item] for k, v in self.value.items(): @@ -614,6 +652,9 @@ class Container(dict): def __str__(self): # type: () -> str return str(self.value) + def __repr__(self): # type: () -> str + return super(Container, self).__repr__() + def __eq__(self, other): # type: (Dict) -> bool if not isinstance(other, dict): return NotImplemented @@ -669,9 +710,17 @@ class OutOfOrderTableProxy(dict): for k, v in item.value.body: self._internal_container.append(k, v) self._tables_map[k] = table_idx + if k is not None: + super(OutOfOrderTableProxy, self).__setitem__(k.key, v) else: self._internal_container.append(key, item) self._map[key] = i + if key is not None: + super(OutOfOrderTableProxy, self).__setitem__(key.key, item) + + @property + def value(self): + return self._internal_container.value def __getitem__(self, key): # type: (Union[Key, str]) -> Any if key not in self._internal_container: @@ -692,6 +741,9 @@ class OutOfOrderTableProxy(dict): else: self._container[key] = item + if key is not None: + super(OutOfOrderTableProxy, self).__setitem__(key, item) + def __delitem__(self, key): # type: (Union[Key, str]) -> None if key in self._map: idx = self._map[key] diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py index c1a4e620..44836363 100644 --- a/pipenv/vendor/tomlkit/exceptions.py +++ b/pipenv/vendor/tomlkit/exceptions.py @@ -1,3 +1,5 @@ +from typing import Optional + class TOMLKitError(Exception): @@ -200,3 +202,20 @@ class KeyAlreadyPresent(TOMLKitError): message = 'Key "{}" already exists.'.format(key) super(KeyAlreadyPresent, self).__init__(message) + + +class InvalidControlChar(ParseError): + def __init__(self, line, col, char, type): # type: (int, int, int, str) -> None + display_code = "\\u00" + + if char < 16: + display_code += "0" + + display_code += str(char) + + message = ( + "Control characters (codes less than 0x1f and 0x7f) are not allowed in {}, " + "use {} instead".format(type, display_code) + ) + + super(InvalidControlChar, self).__init__(line, col, message=message) diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 309fe1d8..a691c162 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -6,6 +6,13 @@ import string from datetime import date from datetime import datetime from datetime import time +from enum import Enum +from typing import Any +from typing import Dict +from typing import Generator +from typing import List +from typing import Optional +from typing import Union from ._compat import PY2 from ._compat import PY38 @@ -14,16 +21,14 @@ from ._compat import long from ._compat import unicode from ._utils import escape_string + if PY2: - from pipenv.vendor.backports.enum import Enum from pipenv.vendor.backports.functools_lru_cache import lru_cache else: - from enum import Enum from functools import lru_cache -from toml.decoder import InlineTableDict -def item(value, _parent=None): +def item(value, _parent=None, _sort_keys=False): from .container import Container if isinstance(value, Item): @@ -37,12 +42,11 @@ def item(value, _parent=None): return Float(value, Trivia(), str(value)) elif isinstance(value, dict): val = Table(Container(), Trivia(), False) - if isinstance(value, InlineTableDict): - val = InlineTable(Container(), Trivia()) - else: - val = Table(Container(), Trivia(), False) - for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): - val[k] = item(v, _parent=val) + for k, v in sorted( + value.items(), + key=lambda i: (isinstance(i[1], dict), i[0] if _sort_keys else 1), + ): + val[k] = item(v, _parent=val, _sort_keys=_sort_keys) return val elif isinstance(value, list): @@ -56,13 +60,14 @@ def item(value, _parent=None): table = Table(Container(), Trivia(), True) for k, _v in sorted( - v.items(), key=lambda i: (isinstance(i[1], dict), i[0]) + v.items(), + key=lambda i: (isinstance(i[1], dict), i[0] if _sort_keys else 1), ): - i = item(_v) + i = item(_v, _sort_keys=_sort_keys) if isinstance(table, InlineTable): i.trivia.trail = "" - table[k] = item(i) + table[k] = item(i, _sort_keys=_sort_keys) v = table @@ -200,7 +205,9 @@ class Key: A key value. """ - def __init__(self, k, t=None, sep=None, dotted=False): # type: (str) -> None + def __init__( + self, k, t=None, sep=None, dotted=False, original=None + ): # type: (str, Optional[KeyType], Optional[str], bool, Optional[str]) -> None if t is None: if any( [c not in string.ascii_letters + string.digits + "-" + "_" for c in k] @@ -215,6 +222,11 @@ class Key: self.sep = sep self.key = k + if original is None: + original = k + + self._original = original + self._dotted = dotted @property @@ -224,8 +236,11 @@ class Key: def is_dotted(self): # type: () -> bool return self._dotted + def is_bare(self): # type: () -> bool + return self.t == KeyType.Bare + def as_string(self): # type: () -> str - return "{}{}{}".format(self.delimiter, self.key, self.delimiter) + return "{}{}{}".format(self.delimiter, self._original, self.delimiter) def __hash__(self): # type: () -> int return hash(self.key) @@ -290,6 +305,9 @@ class Item(object): def is_inline_table(self): # type: () -> bool return isinstance(self, InlineTable) + def is_aot(self): # type: () -> bool + return isinstance(self, AoT) + def _getstate(self, protocol=3): return (self._trivia,) @@ -525,6 +543,12 @@ class Bool(Item): return other == self._value + def __hash__(self): + return hash(self._value) + + def __repr__(self): + return repr(self._value) + class DateTime(Item, datetime): """ @@ -544,7 +568,7 @@ class DateTime(Item, datetime): trivia, raw, **kwargs - ): # type: (int, int, int, int, int, int, int, ..., Trivia, ...) -> datetime + ): # type: (int, int, int, int, int, int, int, Optional[datetime.tzinfo], Trivia, str, Any) -> datetime return datetime.__new__( cls, year, @@ -560,7 +584,7 @@ class DateTime(Item, datetime): def __init__( self, year, month, day, hour, minute, second, microsecond, tzinfo, trivia, raw - ): # type: (int, int, int, int, int, int, int, ..., Trivia) -> None + ): # type: (int, int, int, int, int, int, int, Optional[datetime.tzinfo], Trivia, str) -> None super(DateTime, self).__init__(trivia) self._raw = raw @@ -649,7 +673,7 @@ class Date(Item, date): A date literal. """ - def __new__(cls, year, month, day, *_): # type: (int, int, int, ...) -> date + def __new__(cls, year, month, day, *_): # type: (int, int, int, Any) -> date return date.__new__(cls, year, month, day) def __init__( @@ -705,12 +729,12 @@ class Time(Item, time): def __new__( cls, hour, minute, second, microsecond, tzinfo, *_ - ): # type: (int, int, int, int, ...) -> time + ): # type: (int, int, int, int, Optional[datetime.tzinfo], Any) -> time return time.__new__(cls, hour, minute, second, microsecond, tzinfo) def __init__( self, hour, minute, second, microsecond, tzinfo, trivia, raw - ): # type: (int, int, int, int, Trivia, str) -> None + ): # type: (int, int, int, int, Optional[datetime.tzinfo], Trivia, str) -> None super(Time, self).__init__(trivia) self._raw = raw @@ -743,7 +767,9 @@ class Array(Item, list): An array literal """ - def __init__(self, value, trivia, multiline=False): # type: (list, Trivia) -> None + def __init__( + self, value, trivia, multiline=False + ): # type: (list, Trivia, bool) -> None super(Array, self).__init__(trivia) list.__init__( @@ -761,18 +787,6 @@ class Array(Item, list): def value(self): # type: () -> list return self - def is_homogeneous(self): # type: () -> bool - if not self: - return True - - discriminants = [ - i.discriminant - for i in self._value - if not isinstance(i, (Whitespace, Comment)) - ] - - return len(set(discriminants)) == 1 - def multiline(self, multiline): # type: (bool) -> self self._multiline = multiline @@ -791,7 +805,7 @@ class Array(Item, list): return s - def append(self, _item): # type: () -> None + def append(self, _item): # type: (Any) -> None if self._value: self._value.append(Whitespace(", ")) @@ -800,9 +814,6 @@ class Array(Item, list): self._value.append(it) - if not self.is_homogeneous(): - raise ValueError("Array has mixed types elements") - if not PY2: def clear(self): @@ -868,7 +879,7 @@ class Table(Item, dict): is_super_table=False, name=None, display_name=None, - ): # type: (tomlkit.container.Container, Trivia, bool, ...) -> None + ): # type: (tomlkit.container.Container, Trivia, bool, bool, Optional[str], Optional[str]) -> None super(Table, self).__init__(trivia) self.name = name @@ -961,8 +972,8 @@ class Table(Item, dict): def is_super_table(self): # type: () -> bool return self._is_super_table - def as_string(self, prefix=None): # type: () -> str - return self._value.as_string(prefix=prefix) + def as_string(self): # type: () -> str + return self._value.as_string() # Helpers @@ -1123,7 +1134,7 @@ class InlineTable(Item, dict): buf += "{}{}{}{}{}{}".format( v.trivia.indent, - k.as_string(), + k.as_string() + ("." if k.is_dotted() else ""), k.sep, v.as_string(), v.trivia.comment, @@ -1249,7 +1260,7 @@ class AoT(Item, list): def __init__( self, body, name=None, parsed=False - ): # type: (List[Table], Optional[str]) -> None + ): # type: (List[Table], Optional[str], bool) -> None self.name = name self._body = [] self._parsed = parsed @@ -1294,7 +1305,7 @@ class AoT(Item, list): def as_string(self): # type: () -> str b = "" for table in self._body: - b += table.as_string(prefix=self.name) + b += table.as_string() return b diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py index 13fd9f98..49929954 100644 --- a/pipenv/vendor/tomlkit/parser.py +++ b/pipenv/vendor/tomlkit/parser.py @@ -4,22 +4,29 @@ from __future__ import unicode_literals import re import string +from typing import Any +from typing import Generator +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union + from ._compat import chr from ._compat import decode -from ._utils import _escaped from ._utils import RFC_3339_LOOSE +from ._utils import _escaped from ._utils import parse_rfc3339 from .container import Container from .exceptions import EmptyKeyError from .exceptions import EmptyTableNameError from .exceptions import InternalParserError from .exceptions import InvalidCharInStringError -from .exceptions import InvalidDateTimeError +from .exceptions import InvalidControlChar from .exceptions import InvalidDateError -from .exceptions import InvalidTimeError +from .exceptions import InvalidDateTimeError from .exceptions import InvalidNumberError +from .exceptions import InvalidTimeError from .exceptions import InvalidUnicodeValueError -from .exceptions import MixedArrayTypesError from .exceptions import ParseError from .exceptions import UnexpectedCharError from .exceptions import UnexpectedEofError @@ -48,6 +55,13 @@ from .toml_char import TOMLChar from .toml_document import TOMLDocument +CTRL_I = 0x09 # Tab +CTRL_J = 0x0A # Line feed +CTRL_M = 0x0D # Carriage return +CTRL_CHAR_LIMIT = 0x1F +CHR_DEL = 0x7F + + class Parser: """ Parser for TOML documents. @@ -195,7 +209,7 @@ class Parser: current = "" t = KeyType.Bare parts = 0 - for c in name.strip(): + for c in name: c = TOMLChar(c) if c == ".": @@ -206,7 +220,8 @@ class Parser: if not current: raise self.parse_error() - yield Key(current.strip(), t=t, sep="") + yield Key(current.strip(), t=t, sep="", original=current) + parts += 1 current = "" @@ -228,7 +243,11 @@ class Parser: in_name = False else: - if current and TOMLChar(current[-1]).is_spaces() and not parts: + if ( + current.strip() + and TOMLChar(current[-1]).is_spaces() + and not parts + ): raise self.parse_error() in_name = True @@ -236,14 +255,6 @@ class Parser: continue elif in_name or c.is_bare_key_char(): - if ( - not in_name - and current - and TOMLChar(current[-1]).is_spaces() - and not parts - ): - raise self.parse_error() - current += c elif c.is_spaces(): # A space is only valid at this point @@ -256,7 +267,7 @@ class Parser: raise self.parse_error() if current.strip(): - yield Key(current.strip(), t=t, sep="") + yield Key(current.strip(), t=t, sep="", original=current) def _parse_item(self): # type: () -> Optional[Tuple[Optional[Key], Item]] """ @@ -319,8 +330,13 @@ class Parser: self.inc() # Skip # # The comment itself - while not self.end() and not self._current.is_nl() and self.inc(): - pass + while not self.end() and not self._current.is_nl(): + code = ord(self._current) + if code == CHR_DEL or code <= CTRL_CHAR_LIMIT and code != CTRL_I: + raise self.parse_error(InvalidControlChar, code, "comments") + + if not self.inc(): + break comment = self.extract() self.mark() @@ -360,8 +376,6 @@ class Parser: # Key key = self._parse_key() - if not key.key.strip(): - raise self.parse_error(EmptyKeyError) self.mark() @@ -374,16 +388,20 @@ class Parser: found_equals = True pass - key.sep = self.extract() + if not key.sep: + key.sep = self.extract() + else: + key.sep += self.extract() # Value val = self._parse_value() - # Comment if parse_comment: cws, comment, trail = self._parse_comment_trail() meta = val.trivia - meta.comment_ws = cws + if not meta.comment_ws: + meta.comment_ws = cws + meta.comment = comment meta.trail = trail else: @@ -444,30 +462,64 @@ class Parser: dotted = False self.mark() - while self._current.is_bare_key_char() and self.inc(): + while ( + self._current.is_bare_key_char() or self._current.is_spaces() + ) and self.inc(): pass - key = self.extract() + original = self.extract() + key = original.strip() + if not key: + # Empty key + raise self.parse_error(ParseError, "Empty key found") + + if " " in key: + # Bare key with spaces in it + raise self.parse_error(ParseError, 'Invalid key "{}"'.format(key)) if self._current == ".": self.inc() dotted = True - key += "." + self._parse_key().as_string() + original += "." + self._parse_key().as_string() + key = original.strip() key_type = KeyType.Bare - return Key(key, key_type, "", dotted) + return Key(key, key_type, "", dotted, original=original) def _handle_dotted_key( self, container, key, value ): # type: (Union[Container, Table], Key, Any) -> None - names = tuple(self._split_table_name(key.key)) + names = tuple(self._split_table_name(key.as_string())) name = names[0] name._dotted = True if name in container: - if isinstance(container, Table): - table = container.value.item(name) - else: - table = container.item(name) + if not isinstance(value, Table): + table = Table(Container(True), Trivia(), False, is_super_table=True) + _table = table + for i, _name in enumerate(names[1:]): + if i == len(names) - 2: + _name.sep = key.sep + + _table.append(_name, value) + else: + _name._dotted = True + _table.append( + _name, + Table( + Container(True), + Trivia(), + False, + is_super_table=i < len(names) - 2, + ), + ) + + _table = _table[_name] + + value = table + + container.append(name, value) + + return else: table = Table(Container(True), Trivia(), False, is_super_table=True) if isinstance(container, Table): @@ -483,7 +535,7 @@ class Parser: else: _name._dotted = True if _name in table.value: - table = table.value.item(_name) + table = table.value[_name] else: table.append( _name, @@ -567,7 +619,29 @@ class Parser: if m.group(1): try: dt = parse_rfc3339(raw) - return Date(dt.year, dt.month, dt.day, trivia, raw) + date = Date(dt.year, dt.month, dt.day, trivia, raw) + self.mark() + while self._current not in "\t\n\r#,]}" and self.inc(): + pass + + time_raw = self.extract() + if not time_raw.strip(): + trivia.comment_ws = time_raw + return date + + dt = parse_rfc3339(raw + time_raw) + return DateTime( + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + dt.microsecond, + dt.tzinfo, + trivia, + raw + time_raw, + ) except ValueError: raise self.parse_error(InvalidDateError) @@ -667,10 +741,7 @@ class Parser: except ValueError: pass else: - if res.is_homogeneous(): - return res - - raise self.parse_error(MixedArrayTypesError) + return res def _parse_inline_table(self): # type: () -> InlineTable # consume opening bracket, EOF here is an issue (middle of array) @@ -693,15 +764,25 @@ class Parser: # consume closing bracket, EOF here doesn't matter self.inc() break - if trailing_comma is False: + + if ( + trailing_comma is False + or trailing_comma is None + and self._current == "," + ): + # Either the previous key-value pair was not followed by a comma + # or the table has an unexpected leading comma. raise self.parse_error(UnexpectedCharError, self._current) else: # True: previous key-value pair was followed by a comma - if self._current == "}": + if self._current == "}" or self._current == ",": raise self.parse_error(UnexpectedCharError, self._current) key, val = self._parse_key_value(False) - elems.add(key, val) + if key.is_dotted(): + self._handle_dotted_key(elems, key, val) + else: + elems.add(key, val) # consume trailing whitespace mark = self._idx @@ -728,7 +809,7 @@ class Parser: if ( len(raw) > 1 and raw.startswith("0") - and not raw.startswith(("0.", "0o", "0x", "0b")) + and not raw.startswith(("0.", "0o", "0x", "0b", "0e")) ): return @@ -851,33 +932,52 @@ class Parser: escaped = False # whether the previous key was ESCAPE while True: - if delim.is_singleline() and self._current.is_nl(): - # single line cannot have actual newline characters - raise self.parse_error(InvalidCharInStringError, self._current) + code = ord(self._current) + if ( + delim.is_singleline() + and not escaped + and (code == CHR_DEL or code <= CTRL_CHAR_LIMIT and code != CTRL_I) + ): + raise self.parse_error(InvalidControlChar, code, "strings") + elif ( + delim.is_multiline() + and not escaped + and ( + code == CHR_DEL + or code <= CTRL_CHAR_LIMIT + and code not in [CTRL_I, CTRL_J, CTRL_M] + ) + ): + raise self.parse_error(InvalidControlChar, code, "strings") elif not escaped and self._current == delim.unit: # try to process current as a closing delim original = self.extract() close = "" if delim.is_multiline(): - # try consuming three delims as this would mean the end of - # the string - for last in [False, False, True]: - if self._current != delim.unit: - # Not a triple quote, leave in result as-is. - # Adding back the characters we already consumed - value += close - close = "" # clear the close - break + # Consume the delimiters to see if we are at the end of the string + close = "" + while self._current == delim.unit: + close += self._current + self.inc() - close += delim.unit - - # consume this delim, EOF here is only an issue if this - # is not the third (last) delim character - self.inc(exception=UnexpectedEofError if not last else None) - - if not close: # if there is no close characters, keep parsing + if len(close) < 3: + # Not a triple quote, leave in result as-is. + # Adding back the characters we already consumed + value += close continue + + if len(close) == 3: + # We are at the end of the string + return String(delim, value, original, Trivia()) + + if len(close) >= 6: + raise self.parse_error(InvalidCharInStringError, self._current) + + value += close[:-3] + original += close[:-3] + + return String(delim, value, original, Trivia()) else: # consume the closing delim, we do not care if EOF occurs as # that would simply imply the end of self._src @@ -906,8 +1006,8 @@ class Parser: self.inc(exception=UnexpectedEofError) def _parse_table( - self, parent_name=None - ): # type: (Optional[str]) -> Tuple[Key, Union[Table, AoT]] + self, parent_name=None, parent=None + ): # type: (Optional[str], Optional[Table]) -> Tuple[Key, Union[Table, AoT]] """ Parses a table element. """ @@ -974,6 +1074,9 @@ class Parser: key = Key(name, sep="") name_parts = tuple(self._split_table_name(name)) + if any(" " in part.key.strip() and part.is_bare() for part in name_parts): + raise self.parse_error(ParseError, 'Invalid table name "{}"'.format(name)) + missing_table = False if parent_name: parent_name_parts = tuple(self._split_table_name(parent_name)) @@ -1059,7 +1162,7 @@ class Parser: is_aot_next, name_next = self._peek_table() if self._is_child(name, name_next): - key_next, table_next = self._parse_table(name) + key_next, table_next = self._parse_table(name, table) table.raw_append(key_next, table_next) @@ -1070,7 +1173,7 @@ class Parser: if not self._is_child(name, name_next): break - key_next, table_next = self._parse_table(name) + key_next, table_next = self._parse_table(name, table) table.raw_append(key_next, table_next) @@ -1190,7 +1293,7 @@ class Parser: try: value = chr(int(extracted, 16)) - except ValueError: + except (ValueError, OverflowError): value = None return value, extracted diff --git a/pipenv/vendor/tomlkit/source.py b/pipenv/vendor/tomlkit/source.py index ddb580e4..6a6a2391 100644 --- a/pipenv/vendor/tomlkit/source.py +++ b/pipenv/vendor/tomlkit/source.py @@ -4,12 +4,16 @@ from __future__ import unicode_literals import itertools from copy import copy +from typing import Any +from typing import Optional +from typing import Tuple +from typing import Type from ._compat import PY2 from ._compat import unicode -from .exceptions import UnexpectedEofError -from .exceptions import UnexpectedCharError from .exceptions import ParseError +from .exceptions import UnexpectedCharError +from .exceptions import UnexpectedEofError from .toml_char import TOMLChar @@ -114,7 +118,7 @@ class Source(unicode): """ return self[self._marker : self._idx] - def inc(self, exception=None): # type: (Optional[ParseError.__class__]) -> bool + def inc(self, exception=None): # type: (Optional[Type[ParseError]]) -> bool """ Increments the parser if the end of the input has not been reached. Returns whether or not it was able to advance. @@ -170,7 +174,7 @@ class Source(unicode): def parse_error( self, exception=ParseError, *args - ): # type: (ParseError.__class__, ...) -> ParseError + ): # type: (Type[ParseError], Any) -> ParseError """ Creates a generic "parse error" at the current position. """ diff --git a/pipenv/vendor/tomlkit/toml_char.py b/pipenv/vendor/tomlkit/toml_char.py index 02c55172..1faf0aaa 100644 --- a/pipenv/vendor/tomlkit/toml_char.py +++ b/pipenv/vendor/tomlkit/toml_char.py @@ -3,6 +3,7 @@ import string from ._compat import PY2 from ._compat import unicode + if PY2: from pipenv.vendor.backports.functools_lru_cache import lru_cache else: diff --git a/pipenv/vendor/tomlkit/toml_file.py b/pipenv/vendor/tomlkit/toml_file.py index 631e9959..3b416664 100644 --- a/pipenv/vendor/tomlkit/toml_file.py +++ b/pipenv/vendor/tomlkit/toml_file.py @@ -1,5 +1,8 @@ import io +from typing import Any +from typing import Dict + from .api import loads from .toml_document import TOMLDocument diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 91790fc3..c55bb6c4 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -32,7 +32,7 @@ requirementslib==1.5.13 packaging==20.3 pyparsing==2.4.7 plette==0.2.3 - tomlkit==0.5.11 + tomlkit==0.7.0 shellingham==1.3.2 six==1.14.0 semver==2.9.0 @@ -57,3 +57,4 @@ git+https://github.com/sarugaku/passa.git@master#egg=passa orderedmultidict==1.0.1 dparse==0.5.0 python-dateutil==2.8.1 +tomlkit \ No newline at end of file diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index e55bdfdf..87d93586 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -566,7 +566,10 @@ def packages_missing_licenses( ): if not vendor_dir: vendor_dir = _get_vendor_dir(ctx) - requirements = vendor_dir.joinpath(requirements_file).read_text().splitlines() + if package is not None: + requirements = [package] + else: + requirements = vendor_dir.joinpath(requirements_file).read_text().splitlines() new_requirements = [] LICENSE_EXTS = ("rst", "txt", "APACHE", "BSD", "md") LICENSES = [