Merge pull request #4255 from pypa/bugfix/4226

This commit is contained in:
Dan Ryan
2020-05-19 16:46:44 -04:00
committed by GitHub
73 changed files with 9382 additions and 797 deletions
+1
View File
@@ -0,0 +1 @@
Fixed a bug which prevented resolution of direct URL dependencies which have PEP508 style direct url VCS sub-dependencies with subdirectories.
+1
View File
@@ -0,0 +1 @@
``Requires-Python`` values specifying constraint versions of python starting from ``1.x`` will now be parsed successfully.
+11
View File
@@ -0,0 +1,11 @@
Updated vendored dependencies to latest versions for security and bug fixes:
- **requirementslib** ``1.5.8`` => ``1.5.9``
- **vistir** ``0.5.0`` => ``0.5.1``
- **jinja2** ``2.11.1`` => ``2.11.2``
- **click** ``7.1.1`` => ``7.1.2``
- **dateutil** ``(none)`` => ``2.8.1``
- **backports.functools_lru_cache** ``1.5.0`` => ``1.6.1``
- **enum34** ``1.1.6`` => ``1.1.10``
- **toml** ``0.10.0`` => ``0.10.1``
- **importlib_resources** ``1.4.0`` => ``1.5.0``
+6 -9
View File
@@ -586,7 +586,7 @@ class Resolver(object):
constraints.add(line)
# ensure the top level entry remains as provided
# note that we shouldn't pin versions for editable vcs deps
if (not req.is_vcs or (req.is_vcs and not req.editable)):
if not req.is_vcs:
if req.specifiers:
locked_deps[name]["version"] = req.specifiers
elif parsed_line.setup_info and parsed_line.setup_info.version:
@@ -997,6 +997,8 @@ class Resolver(object):
for req, ireq in reqs:
if (req.vcs and req.editable and not req.is_direct_url):
continue
elif req.normalized_name in self.skipped.keys():
continue
collected_hashes = self.collect_hashes(ireq)
req = req.add_hashes(collected_hashes)
if not collected_hashes and self._should_include_hash(ireq):
@@ -1041,9 +1043,9 @@ def format_requirement_for_lockfile(req, markers_lookup, index_lookup, hashes=No
entry["version"] = pf_entry.lstrip("=")
else:
entry.update(pf_entry)
if version is not None:
if version is not None and not req.is_vcs:
entry["version"] = version
if req.line_instance.is_direct_url:
if req.line_instance.is_direct_url and not req.is_vcs:
entry["file"] = req.req.uri
if hashes:
entry["hashes"] = sorted(set(hashes))
@@ -1054,7 +1056,7 @@ def format_requirement_for_lockfile(req, markers_lookup, index_lookup, hashes=No
entry.update({"markers": markers})
entry = translate_markers(entry)
if req.vcs or req.editable:
for key in ("index", "version"):
for key in ("index", "version", "file"):
try:
del entry[key]
except KeyError:
@@ -1879,11 +1881,6 @@ def get_vcs_deps(
lockfile[name] = requirement.pipfile_entry[1]
lockfile[name]['ref'] = commit_hash
result.append(requirement)
version = requirement.specifiers
if not version and requirement.specifiers:
version = requirement.specifiers
if version:
lockfile[name]['version'] = version
except OSError:
continue
return result, lockfile
+2 -2
View File
@@ -13,8 +13,8 @@ See <http://github.com/ActiveState/appdirs> for details and usage.
# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html
# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
__version_info__ = (1, 4, 3)
__version__ = '.'.join(map(str, __version_info__))
__version__ = "1.4.4"
__version_info__ = tuple(int(segment) for segment in __version__.split("."))
import sys
+4 -3
View File
@@ -4,7 +4,7 @@ import sys as _sys
__all__ = ['Enum', 'IntEnum', 'unique']
version = 1, 1, 6
version = 1, 1, 10
pyver = float('%s.%s' % _sys.version_info[:2])
@@ -183,7 +183,8 @@ class EnumMeta(type):
else:
del classdict['_order_']
if pyver < 3.0:
_order_ = _order_.replace(',', ' ').split()
if isinstance(_order_, basestring):
_order_ = _order_.replace(',', ' ').split()
aliases = [name for name in members if name not in _order_]
_order_ += aliases
@@ -463,7 +464,7 @@ class EnumMeta(type):
_order_.append(member_name)
# only set _order_ in classdict if name/value was not from a mapping
if not isinstance(item, basestring):
classdict['_order_'] = ' '.join(_order_)
classdict['_order_'] = _order_
enum_class = metacls.__new__(metacls, class_name, bases, classdict)
# TODO: replace the frame hack if a blessed way to know the calling
+31 -19
View File
@@ -8,10 +8,12 @@ _CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
@functools.wraps(functools.update_wrapper)
def update_wrapper(wrapper,
wrapped,
assigned = functools.WRAPPER_ASSIGNMENTS,
updated = functools.WRAPPER_UPDATES):
def update_wrapper(
wrapper,
wrapped,
assigned=functools.WRAPPER_ASSIGNMENTS,
updated=functools.WRAPPER_UPDATES,
):
"""
Patch two bugs in functools.update_wrapper.
"""
@@ -34,10 +36,17 @@ class _HashedSeq(list):
return self.hashvalue
def _make_key(args, kwds, typed,
kwd_mark=(object(),),
fasttypes=set([int, str, frozenset, type(None)]),
sorted=sorted, tuple=tuple, type=type, len=len):
def _make_key(
args,
kwds,
typed,
kwd_mark=(object(),),
fasttypes=set([int, str, frozenset, type(None)]),
sorted=sorted,
tuple=tuple,
type=type,
len=len,
):
'Make a cache key from optionally typed positional and keyword arguments'
key = args
if kwds:
@@ -82,16 +91,16 @@ def lru_cache(maxsize=100, typed=False):
def decorating_function(user_function):
cache = dict()
stats = [0, 0] # make statistics updateable non-locally
HITS, MISSES = 0, 1 # names for the stats fields
stats = [0, 0] # make statistics updateable non-locally
HITS, MISSES = 0, 1 # names for the stats fields
make_key = _make_key
cache_get = cache.get # bound method to lookup key or return None
_len = len # localize the global len() function
lock = RLock() # because linkedlist updates aren't threadsafe
root = [] # root of the circular doubly linked list
root[:] = [root, root, None, None] # initialize by pointing to self
nonlocal_root = [root] # make updateable non-locally
PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields
cache_get = cache.get # bound method to lookup key or return None
_len = len # localize the global len() function
lock = RLock() # because linkedlist updates aren't threadsafe
root = [] # root of the circular doubly linked list
root[:] = [root, root, None, None] # initialize by pointing to self
nonlocal_root = [root] # make updateable non-locally
PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields
if maxsize == 0:
@@ -106,7 +115,9 @@ def lru_cache(maxsize=100, typed=False):
def wrapper(*args, **kwds):
# simple caching without ordering or size limit
key = make_key(args, kwds, typed)
result = cache_get(key, root) # root used here as a unique not-found sentinel
result = cache_get(
key, root
) # root used here as a unique not-found sentinel
if result is not root:
stats[HITS] += 1
return result
@@ -123,7 +134,8 @@ def lru_cache(maxsize=100, typed=False):
with lock:
link = cache_get(key)
if link is not None:
# record recent use of the key by moving it to the front of the list
# record recent use of the key by moving it
# to the front of the list
root, = nonlocal_root
link_prev, link_next, key, result = link
link_prev[NEXT] = link_next
+1 -1
View File
@@ -76,4 +76,4 @@ from .utils import open_file
# literals.
disable_unicode_literals_warning = False
__version__ = "7.1.1"
__version__ = "7.1.2"
-4
View File
@@ -174,8 +174,6 @@ if PY2:
iteritems = lambda x: x.iteritems()
range_type = xrange
from pipes import quote as shlex_quote
def is_bytes(x):
return isinstance(x, (buffer, bytearray))
@@ -284,8 +282,6 @@ else:
isidentifier = lambda x: x.isidentifier()
iteritems = lambda x: iter(x.items())
from shlex import quote as shlex_quote
def is_bytes(x):
return isinstance(x, (bytes, memoryview, bytearray))
+9 -13
View File
@@ -17,7 +17,6 @@ from ._compat import int_types
from ._compat import isatty
from ._compat import open_stream
from ._compat import range_type
from ._compat import shlex_quote
from ._compat import strip_ansi
from ._compat import term_len
from ._compat import WIN
@@ -346,10 +345,7 @@ def pager(generator, color=None):
fd, filename = tempfile.mkstemp()
os.close(fd)
try:
if (
hasattr(os, "system")
and os.system("more {}".format(shlex_quote(filename))) == 0
):
if hasattr(os, "system") and os.system('more "{}"'.format(filename)) == 0:
return _pipepager(generator, "more", color)
return _nullpager(stdout, generator, color)
finally:
@@ -418,7 +414,7 @@ def _tempfilepager(generator, cmd, color):
with open_stream(filename, "wb")[0] as f:
f.write(text.encode(encoding))
try:
os.system("{} {}".format(shlex_quote(cmd), shlex_quote(filename)))
os.system('{} "{}"'.format(cmd, filename))
finally:
os.unlink(filename)
@@ -463,9 +459,7 @@ class Editor(object):
environ = None
try:
c = subprocess.Popen(
"{} {}".format(shlex_quote(editor), shlex_quote(filename)),
env=environ,
shell=True,
'{} "{}"'.format(editor, filename), env=environ, shell=True,
)
exit_code = c.wait()
if exit_code != 0:
@@ -536,16 +530,18 @@ def open_url(url, wait=False, locate=False):
elif WIN:
if locate:
url = _unquote_file(url)
args = "explorer /select,{}".format(shlex_quote(url))
args = 'explorer /select,"{}"'.format(_unquote_file(url.replace('"', "")))
else:
args = 'start {} "" {}'.format("/WAIT" if wait else "", shlex_quote(url))
args = 'start {} "" "{}"'.format(
"/WAIT" if wait else "", url.replace('"', "")
)
return os.system(args)
elif CYGWIN:
if locate:
url = _unquote_file(url)
args = "cygstart {}".format(shlex_quote(os.path.dirname(url)))
args = 'cygstart "{}"'.format(os.path.dirname(url).replace('"', ""))
else:
args = "cygstart {} {}".format("-w" if wait else "", shlex_quote(url))
args = 'cygstart {} "{}"'.format("-w" if wait else "", url.replace('"', ""))
return os.system(args)
try:
+54
View File
@@ -0,0 +1,54 @@
Copyright 2017- Paul Ganssle <paul@ganssle.io>
Copyright 2017- dateutil contributors (see AUTHORS file)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
The above license applies to all contributions after 2017-12-01, as well as
all contributions that have been re-licensed (see AUTHORS file for the list of
contributors who have re-licensed their code).
--------------------------------------------------------------------------------
dateutil - Extensions to the standard Python datetime module.
Copyright (c) 2003-2011 - Gustavo Niemeyer <gustavo@niemeyer.net>
Copyright (c) 2012-2014 - Tomi Pieviläinen <tomi.pievilainen@iki.fi>
Copyright (c) 2014-2016 - Yaron de Leeuw <me@jarondl.net>
Copyright (c) 2015- - Paul Ganssle <paul@ganssle.io>
Copyright (c) 2015- - dateutil contributors (see AUTHORS file)
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The above BSD License Applies to all code, even that also covered by Apache 2.0.
+8
View File
@@ -0,0 +1,8 @@
# -*- coding: utf-8 -*-
try:
from ._version import version as __version__
except ImportError:
__version__ = 'unknown'
__all__ = ['easter', 'parser', 'relativedelta', 'rrule', 'tz',
'utils', 'zoneinfo']
+43
View File
@@ -0,0 +1,43 @@
"""
Common code used in multiple modules.
"""
class weekday(object):
__slots__ = ["weekday", "n"]
def __init__(self, weekday, n=None):
self.weekday = weekday
self.n = n
def __call__(self, n):
if n == self.n:
return self
else:
return self.__class__(self.weekday, n)
def __eq__(self, other):
try:
if self.weekday != other.weekday or self.n != other.n:
return False
except AttributeError:
return False
return True
def __hash__(self):
return hash((
self.weekday,
self.n,
))
def __ne__(self, other):
return not (self == other)
def __repr__(self):
s = ("MO", "TU", "WE", "TH", "FR", "SA", "SU")[self.weekday]
if not self.n:
return s
else:
return "%s(%+d)" % (s, self.n)
# vim:ts=4:sw=4:et
+4
View File
@@ -0,0 +1,4 @@
# coding: utf-8
# file generated by setuptools_scm
# don't change, don't track in version control
version = '2.8.1'
+89
View File
@@ -0,0 +1,89 @@
# -*- coding: utf-8 -*-
"""
This module offers a generic easter computing method for any given year, using
Western, Orthodox or Julian algorithms.
"""
import datetime
__all__ = ["easter", "EASTER_JULIAN", "EASTER_ORTHODOX", "EASTER_WESTERN"]
EASTER_JULIAN = 1
EASTER_ORTHODOX = 2
EASTER_WESTERN = 3
def easter(year, method=EASTER_WESTERN):
"""
This method was ported from the work done by GM Arts,
on top of the algorithm by Claus Tondering, which was
based in part on the algorithm of Ouding (1940), as
quoted in "Explanatory Supplement to the Astronomical
Almanac", P. Kenneth Seidelmann, editor.
This algorithm implements three different easter
calculation methods:
1 - Original calculation in Julian calendar, valid in
dates after 326 AD
2 - Original method, with date converted to Gregorian
calendar, valid in years 1583 to 4099
3 - Revised method, in Gregorian calendar, valid in
years 1583 to 4099 as well
These methods are represented by the constants:
* ``EASTER_JULIAN = 1``
* ``EASTER_ORTHODOX = 2``
* ``EASTER_WESTERN = 3``
The default method is method 3.
More about the algorithm may be found at:
`GM Arts: Easter Algorithms <http://www.gmarts.org/index.php?go=415>`_
and
`The Calendar FAQ: Easter <https://www.tondering.dk/claus/cal/easter.php>`_
"""
if not (1 <= method <= 3):
raise ValueError("invalid method")
# g - Golden year - 1
# c - Century
# h - (23 - Epact) mod 30
# i - Number of days from March 21 to Paschal Full Moon
# j - Weekday for PFM (0=Sunday, etc)
# p - Number of days from March 21 to Sunday on or before PFM
# (-6 to 28 methods 1 & 3, to 56 for method 2)
# e - Extra days to add for method 2 (converting Julian
# date to Gregorian date)
y = year
g = y % 19
e = 0
if method < 3:
# Old method
i = (19*g + 15) % 30
j = (y + y//4 + i) % 7
if method == 2:
# Extra dates to convert Julian to Gregorian date
e = 10
if y > 1600:
e = e + y//100 - 16 - (y//100 - 16)//4
else:
# New method
c = y//100
h = (c - c//4 - (8*c + 13)//25 + 19*g + 15) % 30
i = h - (h//28)*(1 - (h//28)*(29//(h + 1))*((21 - g)//11))
j = (y + y//4 + i + 2 - c + c//4) % 7
# p can be from -6 to 56 corresponding to dates 22 March to 23 May
# (later dates apply to method 2, although 23 May never actually occurs)
p = i - j + e
d = 1 + (p + 27 + (p + 6)//40) % 31
m = 3 + (p + 26)//30
return datetime.date(int(y), int(m), int(d))
+61
View File
@@ -0,0 +1,61 @@
# -*- coding: utf-8 -*-
from ._parser import parse, parser, parserinfo, ParserError
from ._parser import DEFAULTPARSER, DEFAULTTZPARSER
from ._parser import UnknownTimezoneWarning
from ._parser import __doc__
from .isoparser import isoparser, isoparse
__all__ = ['parse', 'parser', 'parserinfo',
'isoparse', 'isoparser',
'ParserError',
'UnknownTimezoneWarning']
###
# Deprecate portions of the private interface so that downstream code that
# is improperly relying on it is given *some* notice.
def __deprecated_private_func(f):
from functools import wraps
import warnings
msg = ('{name} is a private function and may break without warning, '
'it will be moved and or renamed in future versions.')
msg = msg.format(name=f.__name__)
@wraps(f)
def deprecated_func(*args, **kwargs):
warnings.warn(msg, DeprecationWarning)
return f(*args, **kwargs)
return deprecated_func
def __deprecate_private_class(c):
import warnings
msg = ('{name} is a private class and may break without warning, '
'it will be moved and or renamed in future versions.')
msg = msg.format(name=c.__name__)
class private_class(c):
__doc__ = c.__doc__
def __init__(self, *args, **kwargs):
warnings.warn(msg, DeprecationWarning)
super(private_class, self).__init__(*args, **kwargs)
private_class.__name__ = c.__name__
return private_class
from ._parser import _timelex, _resultbase
from ._parser import _tzparser, _parsetz
_timelex = __deprecate_private_class(_timelex)
_tzparser = __deprecate_private_class(_tzparser)
_resultbase = __deprecate_private_class(_resultbase)
_parsetz = __deprecated_private_func(_parsetz)
File diff suppressed because it is too large Load Diff
+411
View File
@@ -0,0 +1,411 @@
# -*- coding: utf-8 -*-
"""
This module offers a parser for ISO-8601 strings
It is intended to support all valid date, time and datetime formats per the
ISO-8601 specification.
..versionadded:: 2.7.0
"""
from datetime import datetime, timedelta, time, date
import calendar
from dateutil import tz
from functools import wraps
import re
import six
__all__ = ["isoparse", "isoparser"]
def _takes_ascii(f):
@wraps(f)
def func(self, str_in, *args, **kwargs):
# If it's a stream, read the whole thing
str_in = getattr(str_in, 'read', lambda: str_in)()
# If it's unicode, turn it into bytes, since ISO-8601 only covers ASCII
if isinstance(str_in, six.text_type):
# ASCII is the same in UTF-8
try:
str_in = str_in.encode('ascii')
except UnicodeEncodeError as e:
msg = 'ISO-8601 strings should contain only ASCII characters'
six.raise_from(ValueError(msg), e)
return f(self, str_in, *args, **kwargs)
return func
class isoparser(object):
def __init__(self, sep=None):
"""
:param sep:
A single character that separates date and time portions. If
``None``, the parser will accept any single character.
For strict ISO-8601 adherence, pass ``'T'``.
"""
if sep is not None:
if (len(sep) != 1 or ord(sep) >= 128 or sep in '0123456789'):
raise ValueError('Separator must be a single, non-numeric ' +
'ASCII character')
sep = sep.encode('ascii')
self._sep = sep
@_takes_ascii
def isoparse(self, dt_str):
"""
Parse an ISO-8601 datetime string into a :class:`datetime.datetime`.
An ISO-8601 datetime string consists of a date portion, followed
optionally by a time portion - the date and time portions are separated
by a single character separator, which is ``T`` in the official
standard. Incomplete date formats (such as ``YYYY-MM``) may *not* be
combined with a time portion.
Supported date formats are:
Common:
- ``YYYY``
- ``YYYY-MM`` or ``YYYYMM``
- ``YYYY-MM-DD`` or ``YYYYMMDD``
Uncommon:
- ``YYYY-Www`` or ``YYYYWww`` - ISO week (day defaults to 0)
- ``YYYY-Www-D`` or ``YYYYWwwD`` - ISO week and day
The ISO week and day numbering follows the same logic as
:func:`datetime.date.isocalendar`.
Supported time formats are:
- ``hh``
- ``hh:mm`` or ``hhmm``
- ``hh:mm:ss`` or ``hhmmss``
- ``hh:mm:ss.ssssss`` (Up to 6 sub-second digits)
Midnight is a special case for `hh`, as the standard supports both
00:00 and 24:00 as a representation. The decimal separator can be
either a dot or a comma.
.. caution::
Support for fractional components other than seconds is part of the
ISO-8601 standard, but is not currently implemented in this parser.
Supported time zone offset formats are:
- `Z` (UTC)
- `±HH:MM`
- `±HHMM`
- `±HH`
Offsets will be represented as :class:`dateutil.tz.tzoffset` objects,
with the exception of UTC, which will be represented as
:class:`dateutil.tz.tzutc`. Time zone offsets equivalent to UTC (such
as `+00:00`) will also be represented as :class:`dateutil.tz.tzutc`.
:param dt_str:
A string or stream containing only an ISO-8601 datetime string
:return:
Returns a :class:`datetime.datetime` representing the string.
Unspecified components default to their lowest value.
.. warning::
As of version 2.7.0, the strictness of the parser should not be
considered a stable part of the contract. Any valid ISO-8601 string
that parses correctly with the default settings will continue to
parse correctly in future versions, but invalid strings that
currently fail (e.g. ``2017-01-01T00:00+00:00:00``) are not
guaranteed to continue failing in future versions if they encode
a valid date.
.. versionadded:: 2.7.0
"""
components, pos = self._parse_isodate(dt_str)
if len(dt_str) > pos:
if self._sep is None or dt_str[pos:pos + 1] == self._sep:
components += self._parse_isotime(dt_str[pos + 1:])
else:
raise ValueError('String contains unknown ISO components')
if len(components) > 3 and components[3] == 24:
components[3] = 0
return datetime(*components) + timedelta(days=1)
return datetime(*components)
@_takes_ascii
def parse_isodate(self, datestr):
"""
Parse the date portion of an ISO string.
:param datestr:
The string portion of an ISO string, without a separator
:return:
Returns a :class:`datetime.date` object
"""
components, pos = self._parse_isodate(datestr)
if pos < len(datestr):
raise ValueError('String contains unknown ISO ' +
'components: {}'.format(datestr))
return date(*components)
@_takes_ascii
def parse_isotime(self, timestr):
"""
Parse the time portion of an ISO string.
:param timestr:
The time portion of an ISO string, without a separator
:return:
Returns a :class:`datetime.time` object
"""
components = self._parse_isotime(timestr)
if components[0] == 24:
components[0] = 0
return time(*components)
@_takes_ascii
def parse_tzstr(self, tzstr, zero_as_utc=True):
"""
Parse a valid ISO time zone string.
See :func:`isoparser.isoparse` for details on supported formats.
:param tzstr:
A string representing an ISO time zone offset
:param zero_as_utc:
Whether to return :class:`dateutil.tz.tzutc` for zero-offset zones
:return:
Returns :class:`dateutil.tz.tzoffset` for offsets and
:class:`dateutil.tz.tzutc` for ``Z`` and (if ``zero_as_utc`` is
specified) offsets equivalent to UTC.
"""
return self._parse_tzstr(tzstr, zero_as_utc=zero_as_utc)
# Constants
_DATE_SEP = b'-'
_TIME_SEP = b':'
_FRACTION_REGEX = re.compile(b'[\\.,]([0-9]+)')
def _parse_isodate(self, dt_str):
try:
return self._parse_isodate_common(dt_str)
except ValueError:
return self._parse_isodate_uncommon(dt_str)
def _parse_isodate_common(self, dt_str):
len_str = len(dt_str)
components = [1, 1, 1]
if len_str < 4:
raise ValueError('ISO string too short')
# Year
components[0] = int(dt_str[0:4])
pos = 4
if pos >= len_str:
return components, pos
has_sep = dt_str[pos:pos + 1] == self._DATE_SEP
if has_sep:
pos += 1
# Month
if len_str - pos < 2:
raise ValueError('Invalid common month')
components[1] = int(dt_str[pos:pos + 2])
pos += 2
if pos >= len_str:
if has_sep:
return components, pos
else:
raise ValueError('Invalid ISO format')
if has_sep:
if dt_str[pos:pos + 1] != self._DATE_SEP:
raise ValueError('Invalid separator in ISO string')
pos += 1
# Day
if len_str - pos < 2:
raise ValueError('Invalid common day')
components[2] = int(dt_str[pos:pos + 2])
return components, pos + 2
def _parse_isodate_uncommon(self, dt_str):
if len(dt_str) < 4:
raise ValueError('ISO string too short')
# All ISO formats start with the year
year = int(dt_str[0:4])
has_sep = dt_str[4:5] == self._DATE_SEP
pos = 4 + has_sep # Skip '-' if it's there
if dt_str[pos:pos + 1] == b'W':
# YYYY-?Www-?D?
pos += 1
weekno = int(dt_str[pos:pos + 2])
pos += 2
dayno = 1
if len(dt_str) > pos:
if (dt_str[pos:pos + 1] == self._DATE_SEP) != has_sep:
raise ValueError('Inconsistent use of dash separator')
pos += has_sep
dayno = int(dt_str[pos:pos + 1])
pos += 1
base_date = self._calculate_weekdate(year, weekno, dayno)
else:
# YYYYDDD or YYYY-DDD
if len(dt_str) - pos < 3:
raise ValueError('Invalid ordinal day')
ordinal_day = int(dt_str[pos:pos + 3])
pos += 3
if ordinal_day < 1 or ordinal_day > (365 + calendar.isleap(year)):
raise ValueError('Invalid ordinal day' +
' {} for year {}'.format(ordinal_day, year))
base_date = date(year, 1, 1) + timedelta(days=ordinal_day - 1)
components = [base_date.year, base_date.month, base_date.day]
return components, pos
def _calculate_weekdate(self, year, week, day):
"""
Calculate the day of corresponding to the ISO year-week-day calendar.
This function is effectively the inverse of
:func:`datetime.date.isocalendar`.
:param year:
The year in the ISO calendar
:param week:
The week in the ISO calendar - range is [1, 53]
:param day:
The day in the ISO calendar - range is [1 (MON), 7 (SUN)]
:return:
Returns a :class:`datetime.date`
"""
if not 0 < week < 54:
raise ValueError('Invalid week: {}'.format(week))
if not 0 < day < 8: # Range is 1-7
raise ValueError('Invalid weekday: {}'.format(day))
# Get week 1 for the specific year:
jan_4 = date(year, 1, 4) # Week 1 always has January 4th in it
week_1 = jan_4 - timedelta(days=jan_4.isocalendar()[2] - 1)
# Now add the specific number of weeks and days to get what we want
week_offset = (week - 1) * 7 + (day - 1)
return week_1 + timedelta(days=week_offset)
def _parse_isotime(self, timestr):
len_str = len(timestr)
components = [0, 0, 0, 0, None]
pos = 0
comp = -1
if len(timestr) < 2:
raise ValueError('ISO time too short')
has_sep = len_str >= 3 and timestr[2:3] == self._TIME_SEP
while pos < len_str and comp < 5:
comp += 1
if timestr[pos:pos + 1] in b'-+Zz':
# Detect time zone boundary
components[-1] = self._parse_tzstr(timestr[pos:])
pos = len_str
break
if comp < 3:
# Hour, minute, second
components[comp] = int(timestr[pos:pos + 2])
pos += 2
if (has_sep and pos < len_str and
timestr[pos:pos + 1] == self._TIME_SEP):
pos += 1
if comp == 3:
# Fraction of a second
frac = self._FRACTION_REGEX.match(timestr[pos:])
if not frac:
continue
us_str = frac.group(1)[:6] # Truncate to microseconds
components[comp] = int(us_str) * 10**(6 - len(us_str))
pos += len(frac.group())
if pos < len_str:
raise ValueError('Unused components in ISO string')
if components[0] == 24:
# Standard supports 00:00 and 24:00 as representations of midnight
if any(component != 0 for component in components[1:4]):
raise ValueError('Hour may only be 24 at 24:00:00.000')
return components
def _parse_tzstr(self, tzstr, zero_as_utc=True):
if tzstr == b'Z' or tzstr == b'z':
return tz.UTC
if len(tzstr) not in {3, 5, 6}:
raise ValueError('Time zone offset must be 1, 3, 5 or 6 characters')
if tzstr[0:1] == b'-':
mult = -1
elif tzstr[0:1] == b'+':
mult = 1
else:
raise ValueError('Time zone offset requires sign')
hours = int(tzstr[1:3])
if len(tzstr) == 3:
minutes = 0
else:
minutes = int(tzstr[(4 if tzstr[3:4] == self._TIME_SEP else 3):])
if zero_as_utc and hours == 0 and minutes == 0:
return tz.UTC
else:
if minutes > 59:
raise ValueError('Invalid minutes in time zone offset')
if hours > 23:
raise ValueError('Invalid hours in time zone offset')
return tz.tzoffset(None, mult * (hours * 60 + minutes) * 60)
DEFAULT_ISOPARSER = isoparser()
isoparse = DEFAULT_ISOPARSER.isoparse
+599
View File
@@ -0,0 +1,599 @@
# -*- coding: utf-8 -*-
import datetime
import calendar
import operator
from math import copysign
from six import integer_types
from warnings import warn
from ._common import weekday
MO, TU, WE, TH, FR, SA, SU = weekdays = tuple(weekday(x) for x in range(7))
__all__ = ["relativedelta", "MO", "TU", "WE", "TH", "FR", "SA", "SU"]
class relativedelta(object):
"""
The relativedelta type is designed to be applied to an existing datetime and
can replace specific components of that datetime, or represents an interval
of time.
It is based on the specification of the excellent work done by M.-A. Lemburg
in his
`mx.DateTime <https://www.egenix.com/products/python/mxBase/mxDateTime/>`_ extension.
However, notice that this type does *NOT* implement the same algorithm as
his work. Do *NOT* expect it to behave like mx.DateTime's counterpart.
There are two different ways to build a relativedelta instance. The
first one is passing it two date/datetime classes::
relativedelta(datetime1, datetime2)
The second one is passing it any number of the following keyword arguments::
relativedelta(arg1=x,arg2=y,arg3=z...)
year, month, day, hour, minute, second, microsecond:
Absolute information (argument is singular); adding or subtracting a
relativedelta with absolute information does not perform an arithmetic
operation, but rather REPLACES the corresponding value in the
original datetime with the value(s) in relativedelta.
years, months, weeks, days, hours, minutes, seconds, microseconds:
Relative information, may be negative (argument is plural); adding
or subtracting a relativedelta with relative information performs
the corresponding arithmetic operation on the original datetime value
with the information in the relativedelta.
weekday:
One of the weekday instances (MO, TU, etc) available in the
relativedelta module. These instances may receive a parameter N,
specifying the Nth weekday, which could be positive or negative
(like MO(+1) or MO(-2)). Not specifying it is the same as specifying
+1. You can also use an integer, where 0=MO. This argument is always
relative e.g. if the calculated date is already Monday, using MO(1)
or MO(-1) won't change the day. To effectively make it absolute, use
it in combination with the day argument (e.g. day=1, MO(1) for first
Monday of the month).
leapdays:
Will add given days to the date found, if year is a leap
year, and the date found is post 28 of february.
yearday, nlyearday:
Set the yearday or the non-leap year day (jump leap days).
These are converted to day/month/leapdays information.
There are relative and absolute forms of the keyword
arguments. The plural is relative, and the singular is
absolute. For each argument in the order below, the absolute form
is applied first (by setting each attribute to that value) and
then the relative form (by adding the value to the attribute).
The order of attributes considered when this relativedelta is
added to a datetime is:
1. Year
2. Month
3. Day
4. Hours
5. Minutes
6. Seconds
7. Microseconds
Finally, weekday is applied, using the rule described above.
For example
>>> from datetime import datetime
>>> from dateutil.relativedelta import relativedelta, MO
>>> dt = datetime(2018, 4, 9, 13, 37, 0)
>>> delta = relativedelta(hours=25, day=1, weekday=MO(1))
>>> dt + delta
datetime.datetime(2018, 4, 2, 14, 37)
First, the day is set to 1 (the first of the month), then 25 hours
are added, to get to the 2nd day and 14th hour, finally the
weekday is applied, but since the 2nd is already a Monday there is
no effect.
"""
def __init__(self, dt1=None, dt2=None,
years=0, months=0, days=0, leapdays=0, weeks=0,
hours=0, minutes=0, seconds=0, microseconds=0,
year=None, month=None, day=None, weekday=None,
yearday=None, nlyearday=None,
hour=None, minute=None, second=None, microsecond=None):
if dt1 and dt2:
# datetime is a subclass of date. So both must be date
if not (isinstance(dt1, datetime.date) and
isinstance(dt2, datetime.date)):
raise TypeError("relativedelta only diffs datetime/date")
# We allow two dates, or two datetimes, so we coerce them to be
# of the same type
if (isinstance(dt1, datetime.datetime) !=
isinstance(dt2, datetime.datetime)):
if not isinstance(dt1, datetime.datetime):
dt1 = datetime.datetime.fromordinal(dt1.toordinal())
elif not isinstance(dt2, datetime.datetime):
dt2 = datetime.datetime.fromordinal(dt2.toordinal())
self.years = 0
self.months = 0
self.days = 0
self.leapdays = 0
self.hours = 0
self.minutes = 0
self.seconds = 0
self.microseconds = 0
self.year = None
self.month = None
self.day = None
self.weekday = None
self.hour = None
self.minute = None
self.second = None
self.microsecond = None
self._has_time = 0
# Get year / month delta between the two
months = (dt1.year - dt2.year) * 12 + (dt1.month - dt2.month)
self._set_months(months)
# Remove the year/month delta so the timedelta is just well-defined
# time units (seconds, days and microseconds)
dtm = self.__radd__(dt2)
# If we've overshot our target, make an adjustment
if dt1 < dt2:
compare = operator.gt
increment = 1
else:
compare = operator.lt
increment = -1
while compare(dt1, dtm):
months += increment
self._set_months(months)
dtm = self.__radd__(dt2)
# Get the timedelta between the "months-adjusted" date and dt1
delta = dt1 - dtm
self.seconds = delta.seconds + delta.days * 86400
self.microseconds = delta.microseconds
else:
# Check for non-integer values in integer-only quantities
if any(x is not None and x != int(x) for x in (years, months)):
raise ValueError("Non-integer years and months are "
"ambiguous and not currently supported.")
# Relative information
self.years = int(years)
self.months = int(months)
self.days = days + weeks * 7
self.leapdays = leapdays
self.hours = hours
self.minutes = minutes
self.seconds = seconds
self.microseconds = microseconds
# Absolute information
self.year = year
self.month = month
self.day = day
self.hour = hour
self.minute = minute
self.second = second
self.microsecond = microsecond
if any(x is not None and int(x) != x
for x in (year, month, day, hour,
minute, second, microsecond)):
# For now we'll deprecate floats - later it'll be an error.
warn("Non-integer value passed as absolute information. " +
"This is not a well-defined condition and will raise " +
"errors in future versions.", DeprecationWarning)
if isinstance(weekday, integer_types):
self.weekday = weekdays[weekday]
else:
self.weekday = weekday
yday = 0
if nlyearday:
yday = nlyearday
elif yearday:
yday = yearday
if yearday > 59:
self.leapdays = -1
if yday:
ydayidx = [31, 59, 90, 120, 151, 181, 212,
243, 273, 304, 334, 366]
for idx, ydays in enumerate(ydayidx):
if yday <= ydays:
self.month = idx+1
if idx == 0:
self.day = yday
else:
self.day = yday-ydayidx[idx-1]
break
else:
raise ValueError("invalid year day (%d)" % yday)
self._fix()
def _fix(self):
if abs(self.microseconds) > 999999:
s = _sign(self.microseconds)
div, mod = divmod(self.microseconds * s, 1000000)
self.microseconds = mod * s
self.seconds += div * s
if abs(self.seconds) > 59:
s = _sign(self.seconds)
div, mod = divmod(self.seconds * s, 60)
self.seconds = mod * s
self.minutes += div * s
if abs(self.minutes) > 59:
s = _sign(self.minutes)
div, mod = divmod(self.minutes * s, 60)
self.minutes = mod * s
self.hours += div * s
if abs(self.hours) > 23:
s = _sign(self.hours)
div, mod = divmod(self.hours * s, 24)
self.hours = mod * s
self.days += div * s
if abs(self.months) > 11:
s = _sign(self.months)
div, mod = divmod(self.months * s, 12)
self.months = mod * s
self.years += div * s
if (self.hours or self.minutes or self.seconds or self.microseconds
or self.hour is not None or self.minute is not None or
self.second is not None or self.microsecond is not None):
self._has_time = 1
else:
self._has_time = 0
@property
def weeks(self):
return int(self.days / 7.0)
@weeks.setter
def weeks(self, value):
self.days = self.days - (self.weeks * 7) + value * 7
def _set_months(self, months):
self.months = months
if abs(self.months) > 11:
s = _sign(self.months)
div, mod = divmod(self.months * s, 12)
self.months = mod * s
self.years = div * s
else:
self.years = 0
def normalized(self):
"""
Return a version of this object represented entirely using integer
values for the relative attributes.
>>> relativedelta(days=1.5, hours=2).normalized()
relativedelta(days=+1, hours=+14)
:return:
Returns a :class:`dateutil.relativedelta.relativedelta` object.
"""
# Cascade remainders down (rounding each to roughly nearest microsecond)
days = int(self.days)
hours_f = round(self.hours + 24 * (self.days - days), 11)
hours = int(hours_f)
minutes_f = round(self.minutes + 60 * (hours_f - hours), 10)
minutes = int(minutes_f)
seconds_f = round(self.seconds + 60 * (minutes_f - minutes), 8)
seconds = int(seconds_f)
microseconds = round(self.microseconds + 1e6 * (seconds_f - seconds))
# Constructor carries overflow back up with call to _fix()
return self.__class__(years=self.years, months=self.months,
days=days, hours=hours, minutes=minutes,
seconds=seconds, microseconds=microseconds,
leapdays=self.leapdays, year=self.year,
month=self.month, day=self.day,
weekday=self.weekday, hour=self.hour,
minute=self.minute, second=self.second,
microsecond=self.microsecond)
def __add__(self, other):
if isinstance(other, relativedelta):
return self.__class__(years=other.years + self.years,
months=other.months + self.months,
days=other.days + self.days,
hours=other.hours + self.hours,
minutes=other.minutes + self.minutes,
seconds=other.seconds + self.seconds,
microseconds=(other.microseconds +
self.microseconds),
leapdays=other.leapdays or self.leapdays,
year=(other.year if other.year is not None
else self.year),
month=(other.month if other.month is not None
else self.month),
day=(other.day if other.day is not None
else self.day),
weekday=(other.weekday if other.weekday is not None
else self.weekday),
hour=(other.hour if other.hour is not None
else self.hour),
minute=(other.minute if other.minute is not None
else self.minute),
second=(other.second if other.second is not None
else self.second),
microsecond=(other.microsecond if other.microsecond
is not None else
self.microsecond))
if isinstance(other, datetime.timedelta):
return self.__class__(years=self.years,
months=self.months,
days=self.days + other.days,
hours=self.hours,
minutes=self.minutes,
seconds=self.seconds + other.seconds,
microseconds=self.microseconds + other.microseconds,
leapdays=self.leapdays,
year=self.year,
month=self.month,
day=self.day,
weekday=self.weekday,
hour=self.hour,
minute=self.minute,
second=self.second,
microsecond=self.microsecond)
if not isinstance(other, datetime.date):
return NotImplemented
elif self._has_time and not isinstance(other, datetime.datetime):
other = datetime.datetime.fromordinal(other.toordinal())
year = (self.year or other.year)+self.years
month = self.month or other.month
if self.months:
assert 1 <= abs(self.months) <= 12
month += self.months
if month > 12:
year += 1
month -= 12
elif month < 1:
year -= 1
month += 12
day = min(calendar.monthrange(year, month)[1],
self.day or other.day)
repl = {"year": year, "month": month, "day": day}
for attr in ["hour", "minute", "second", "microsecond"]:
value = getattr(self, attr)
if value is not None:
repl[attr] = value
days = self.days
if self.leapdays and month > 2 and calendar.isleap(year):
days += self.leapdays
ret = (other.replace(**repl)
+ datetime.timedelta(days=days,
hours=self.hours,
minutes=self.minutes,
seconds=self.seconds,
microseconds=self.microseconds))
if self.weekday:
weekday, nth = self.weekday.weekday, self.weekday.n or 1
jumpdays = (abs(nth) - 1) * 7
if nth > 0:
jumpdays += (7 - ret.weekday() + weekday) % 7
else:
jumpdays += (ret.weekday() - weekday) % 7
jumpdays *= -1
ret += datetime.timedelta(days=jumpdays)
return ret
def __radd__(self, other):
return self.__add__(other)
def __rsub__(self, other):
return self.__neg__().__radd__(other)
def __sub__(self, other):
if not isinstance(other, relativedelta):
return NotImplemented # In case the other object defines __rsub__
return self.__class__(years=self.years - other.years,
months=self.months - other.months,
days=self.days - other.days,
hours=self.hours - other.hours,
minutes=self.minutes - other.minutes,
seconds=self.seconds - other.seconds,
microseconds=self.microseconds - other.microseconds,
leapdays=self.leapdays or other.leapdays,
year=(self.year if self.year is not None
else other.year),
month=(self.month if self.month is not None else
other.month),
day=(self.day if self.day is not None else
other.day),
weekday=(self.weekday if self.weekday is not None else
other.weekday),
hour=(self.hour if self.hour is not None else
other.hour),
minute=(self.minute if self.minute is not None else
other.minute),
second=(self.second if self.second is not None else
other.second),
microsecond=(self.microsecond if self.microsecond
is not None else
other.microsecond))
def __abs__(self):
return self.__class__(years=abs(self.years),
months=abs(self.months),
days=abs(self.days),
hours=abs(self.hours),
minutes=abs(self.minutes),
seconds=abs(self.seconds),
microseconds=abs(self.microseconds),
leapdays=self.leapdays,
year=self.year,
month=self.month,
day=self.day,
weekday=self.weekday,
hour=self.hour,
minute=self.minute,
second=self.second,
microsecond=self.microsecond)
def __neg__(self):
return self.__class__(years=-self.years,
months=-self.months,
days=-self.days,
hours=-self.hours,
minutes=-self.minutes,
seconds=-self.seconds,
microseconds=-self.microseconds,
leapdays=self.leapdays,
year=self.year,
month=self.month,
day=self.day,
weekday=self.weekday,
hour=self.hour,
minute=self.minute,
second=self.second,
microsecond=self.microsecond)
def __bool__(self):
return not (not self.years and
not self.months and
not self.days and
not self.hours and
not self.minutes and
not self.seconds and
not self.microseconds and
not self.leapdays and
self.year is None and
self.month is None and
self.day is None and
self.weekday is None and
self.hour is None and
self.minute is None and
self.second is None and
self.microsecond is None)
# Compatibility with Python 2.x
__nonzero__ = __bool__
def __mul__(self, other):
try:
f = float(other)
except TypeError:
return NotImplemented
return self.__class__(years=int(self.years * f),
months=int(self.months * f),
days=int(self.days * f),
hours=int(self.hours * f),
minutes=int(self.minutes * f),
seconds=int(self.seconds * f),
microseconds=int(self.microseconds * f),
leapdays=self.leapdays,
year=self.year,
month=self.month,
day=self.day,
weekday=self.weekday,
hour=self.hour,
minute=self.minute,
second=self.second,
microsecond=self.microsecond)
__rmul__ = __mul__
def __eq__(self, other):
if not isinstance(other, relativedelta):
return NotImplemented
if self.weekday or other.weekday:
if not self.weekday or not other.weekday:
return False
if self.weekday.weekday != other.weekday.weekday:
return False
n1, n2 = self.weekday.n, other.weekday.n
if n1 != n2 and not ((not n1 or n1 == 1) and (not n2 or n2 == 1)):
return False
return (self.years == other.years and
self.months == other.months and
self.days == other.days and
self.hours == other.hours and
self.minutes == other.minutes and
self.seconds == other.seconds and
self.microseconds == other.microseconds and
self.leapdays == other.leapdays and
self.year == other.year and
self.month == other.month and
self.day == other.day and
self.hour == other.hour and
self.minute == other.minute and
self.second == other.second and
self.microsecond == other.microsecond)
def __hash__(self):
return hash((
self.weekday,
self.years,
self.months,
self.days,
self.hours,
self.minutes,
self.seconds,
self.microseconds,
self.leapdays,
self.year,
self.month,
self.day,
self.hour,
self.minute,
self.second,
self.microsecond,
))
def __ne__(self, other):
return not self.__eq__(other)
def __div__(self, other):
try:
reciprocal = 1 / float(other)
except TypeError:
return NotImplemented
return self.__mul__(reciprocal)
__truediv__ = __div__
def __repr__(self):
l = []
for attr in ["years", "months", "days", "leapdays",
"hours", "minutes", "seconds", "microseconds"]:
value = getattr(self, attr)
if value:
l.append("{attr}={value:+g}".format(attr=attr, value=value))
for attr in ["year", "month", "day", "weekday",
"hour", "minute", "second", "microsecond"]:
value = getattr(self, attr)
if value is not None:
l.append("{attr}={value}".format(attr=attr, value=repr(value)))
return "{classname}({attrs})".format(classname=self.__class__.__name__,
attrs=", ".join(l))
def _sign(x):
return int(copysign(1, x))
# vim:ts=4:sw=4:et
+1735
View File
File diff suppressed because it is too large Load Diff
+12
View File
@@ -0,0 +1,12 @@
# -*- coding: utf-8 -*-
from .tz import *
from .tz import __doc__
__all__ = ["tzutc", "tzoffset", "tzlocal", "tzfile", "tzrange",
"tzstr", "tzical", "tzwin", "tzwinlocal", "gettz",
"enfold", "datetime_ambiguous", "datetime_exists",
"resolve_imaginary", "UTC", "DeprecatedTzFormatWarning"]
class DeprecatedTzFormatWarning(Warning):
"""Warning raised when time zones are parsed from deprecated formats."""
+419
View File
@@ -0,0 +1,419 @@
from six import PY2
from functools import wraps
from datetime import datetime, timedelta, tzinfo
ZERO = timedelta(0)
__all__ = ['tzname_in_python2', 'enfold']
def tzname_in_python2(namefunc):
"""Change unicode output into bytestrings in Python 2
tzname() API changed in Python 3. It used to return bytes, but was changed
to unicode strings
"""
if PY2:
@wraps(namefunc)
def adjust_encoding(*args, **kwargs):
name = namefunc(*args, **kwargs)
if name is not None:
name = name.encode()
return name
return adjust_encoding
else:
return namefunc
# The following is adapted from Alexander Belopolsky's tz library
# https://github.com/abalkin/tz
if hasattr(datetime, 'fold'):
# This is the pre-python 3.6 fold situation
def enfold(dt, fold=1):
"""
Provides a unified interface for assigning the ``fold`` attribute to
datetimes both before and after the implementation of PEP-495.
:param fold:
The value for the ``fold`` attribute in the returned datetime. This
should be either 0 or 1.
:return:
Returns an object for which ``getattr(dt, 'fold', 0)`` returns
``fold`` for all versions of Python. In versions prior to
Python 3.6, this is a ``_DatetimeWithFold`` object, which is a
subclass of :py:class:`datetime.datetime` with the ``fold``
attribute added, if ``fold`` is 1.
.. versionadded:: 2.6.0
"""
return dt.replace(fold=fold)
else:
class _DatetimeWithFold(datetime):
"""
This is a class designed to provide a PEP 495-compliant interface for
Python versions before 3.6. It is used only for dates in a fold, so
the ``fold`` attribute is fixed at ``1``.
.. versionadded:: 2.6.0
"""
__slots__ = ()
def replace(self, *args, **kwargs):
"""
Return a datetime with the same attributes, except for those
attributes given new values by whichever keyword arguments are
specified. Note that tzinfo=None can be specified to create a naive
datetime from an aware datetime with no conversion of date and time
data.
This is reimplemented in ``_DatetimeWithFold`` because pypy3 will
return a ``datetime.datetime`` even if ``fold`` is unchanged.
"""
argnames = (
'year', 'month', 'day', 'hour', 'minute', 'second',
'microsecond', 'tzinfo'
)
for arg, argname in zip(args, argnames):
if argname in kwargs:
raise TypeError('Duplicate argument: {}'.format(argname))
kwargs[argname] = arg
for argname in argnames:
if argname not in kwargs:
kwargs[argname] = getattr(self, argname)
dt_class = self.__class__ if kwargs.get('fold', 1) else datetime
return dt_class(**kwargs)
@property
def fold(self):
return 1
def enfold(dt, fold=1):
"""
Provides a unified interface for assigning the ``fold`` attribute to
datetimes both before and after the implementation of PEP-495.
:param fold:
The value for the ``fold`` attribute in the returned datetime. This
should be either 0 or 1.
:return:
Returns an object for which ``getattr(dt, 'fold', 0)`` returns
``fold`` for all versions of Python. In versions prior to
Python 3.6, this is a ``_DatetimeWithFold`` object, which is a
subclass of :py:class:`datetime.datetime` with the ``fold``
attribute added, if ``fold`` is 1.
.. versionadded:: 2.6.0
"""
if getattr(dt, 'fold', 0) == fold:
return dt
args = dt.timetuple()[:6]
args += (dt.microsecond, dt.tzinfo)
if fold:
return _DatetimeWithFold(*args)
else:
return datetime(*args)
def _validate_fromutc_inputs(f):
"""
The CPython version of ``fromutc`` checks that the input is a ``datetime``
object and that ``self`` is attached as its ``tzinfo``.
"""
@wraps(f)
def fromutc(self, dt):
if not isinstance(dt, datetime):
raise TypeError("fromutc() requires a datetime argument")
if dt.tzinfo is not self:
raise ValueError("dt.tzinfo is not self")
return f(self, dt)
return fromutc
class _tzinfo(tzinfo):
"""
Base class for all ``dateutil`` ``tzinfo`` objects.
"""
def is_ambiguous(self, dt):
"""
Whether or not the "wall time" of a given datetime is ambiguous in this
zone.
:param dt:
A :py:class:`datetime.datetime`, naive or time zone aware.
:return:
Returns ``True`` if ambiguous, ``False`` otherwise.
.. versionadded:: 2.6.0
"""
dt = dt.replace(tzinfo=self)
wall_0 = enfold(dt, fold=0)
wall_1 = enfold(dt, fold=1)
same_offset = wall_0.utcoffset() == wall_1.utcoffset()
same_dt = wall_0.replace(tzinfo=None) == wall_1.replace(tzinfo=None)
return same_dt and not same_offset
def _fold_status(self, dt_utc, dt_wall):
"""
Determine the fold status of a "wall" datetime, given a representation
of the same datetime as a (naive) UTC datetime. This is calculated based
on the assumption that ``dt.utcoffset() - dt.dst()`` is constant for all
datetimes, and that this offset is the actual number of hours separating
``dt_utc`` and ``dt_wall``.
:param dt_utc:
Representation of the datetime as UTC
:param dt_wall:
Representation of the datetime as "wall time". This parameter must
either have a `fold` attribute or have a fold-naive
:class:`datetime.tzinfo` attached, otherwise the calculation may
fail.
"""
if self.is_ambiguous(dt_wall):
delta_wall = dt_wall - dt_utc
_fold = int(delta_wall == (dt_utc.utcoffset() - dt_utc.dst()))
else:
_fold = 0
return _fold
def _fold(self, dt):
return getattr(dt, 'fold', 0)
def _fromutc(self, dt):
"""
Given a timezone-aware datetime in a given timezone, calculates a
timezone-aware datetime in a new timezone.
Since this is the one time that we *know* we have an unambiguous
datetime object, we take this opportunity to determine whether the
datetime is ambiguous and in a "fold" state (e.g. if it's the first
occurrence, chronologically, of the ambiguous datetime).
:param dt:
A timezone-aware :class:`datetime.datetime` object.
"""
# Re-implement the algorithm from Python's datetime.py
dtoff = dt.utcoffset()
if dtoff is None:
raise ValueError("fromutc() requires a non-None utcoffset() "
"result")
# The original datetime.py code assumes that `dst()` defaults to
# zero during ambiguous times. PEP 495 inverts this presumption, so
# for pre-PEP 495 versions of python, we need to tweak the algorithm.
dtdst = dt.dst()
if dtdst is None:
raise ValueError("fromutc() requires a non-None dst() result")
delta = dtoff - dtdst
dt += delta
# Set fold=1 so we can default to being in the fold for
# ambiguous dates.
dtdst = enfold(dt, fold=1).dst()
if dtdst is None:
raise ValueError("fromutc(): dt.dst gave inconsistent "
"results; cannot convert")
return dt + dtdst
@_validate_fromutc_inputs
def fromutc(self, dt):
"""
Given a timezone-aware datetime in a given timezone, calculates a
timezone-aware datetime in a new timezone.
Since this is the one time that we *know* we have an unambiguous
datetime object, we take this opportunity to determine whether the
datetime is ambiguous and in a "fold" state (e.g. if it's the first
occurrence, chronologically, of the ambiguous datetime).
:param dt:
A timezone-aware :class:`datetime.datetime` object.
"""
dt_wall = self._fromutc(dt)
# Calculate the fold status given the two datetimes.
_fold = self._fold_status(dt, dt_wall)
# Set the default fold value for ambiguous dates
return enfold(dt_wall, fold=_fold)
class tzrangebase(_tzinfo):
"""
This is an abstract base class for time zones represented by an annual
transition into and out of DST. Child classes should implement the following
methods:
* ``__init__(self, *args, **kwargs)``
* ``transitions(self, year)`` - this is expected to return a tuple of
datetimes representing the DST on and off transitions in standard
time.
A fully initialized ``tzrangebase`` subclass should also provide the
following attributes:
* ``hasdst``: Boolean whether or not the zone uses DST.
* ``_dst_offset`` / ``_std_offset``: :class:`datetime.timedelta` objects
representing the respective UTC offsets.
* ``_dst_abbr`` / ``_std_abbr``: Strings representing the timezone short
abbreviations in DST and STD, respectively.
* ``_hasdst``: Whether or not the zone has DST.
.. versionadded:: 2.6.0
"""
def __init__(self):
raise NotImplementedError('tzrangebase is an abstract base class')
def utcoffset(self, dt):
isdst = self._isdst(dt)
if isdst is None:
return None
elif isdst:
return self._dst_offset
else:
return self._std_offset
def dst(self, dt):
isdst = self._isdst(dt)
if isdst is None:
return None
elif isdst:
return self._dst_base_offset
else:
return ZERO
@tzname_in_python2
def tzname(self, dt):
if self._isdst(dt):
return self._dst_abbr
else:
return self._std_abbr
def fromutc(self, dt):
""" Given a datetime in UTC, return local time """
if not isinstance(dt, datetime):
raise TypeError("fromutc() requires a datetime argument")
if dt.tzinfo is not self:
raise ValueError("dt.tzinfo is not self")
# Get transitions - if there are none, fixed offset
transitions = self.transitions(dt.year)
if transitions is None:
return dt + self.utcoffset(dt)
# Get the transition times in UTC
dston, dstoff = transitions
dston -= self._std_offset
dstoff -= self._std_offset
utc_transitions = (dston, dstoff)
dt_utc = dt.replace(tzinfo=None)
isdst = self._naive_isdst(dt_utc, utc_transitions)
if isdst:
dt_wall = dt + self._dst_offset
else:
dt_wall = dt + self._std_offset
_fold = int(not isdst and self.is_ambiguous(dt_wall))
return enfold(dt_wall, fold=_fold)
def is_ambiguous(self, dt):
"""
Whether or not the "wall time" of a given datetime is ambiguous in this
zone.
:param dt:
A :py:class:`datetime.datetime`, naive or time zone aware.
:return:
Returns ``True`` if ambiguous, ``False`` otherwise.
.. versionadded:: 2.6.0
"""
if not self.hasdst:
return False
start, end = self.transitions(dt.year)
dt = dt.replace(tzinfo=None)
return (end <= dt < end + self._dst_base_offset)
def _isdst(self, dt):
if not self.hasdst:
return False
elif dt is None:
return None
transitions = self.transitions(dt.year)
if transitions is None:
return False
dt = dt.replace(tzinfo=None)
isdst = self._naive_isdst(dt, transitions)
# Handle ambiguous dates
if not isdst and self.is_ambiguous(dt):
return not self._fold(dt)
else:
return isdst
def _naive_isdst(self, dt, transitions):
dston, dstoff = transitions
dt = dt.replace(tzinfo=None)
if dston < dstoff:
isdst = dston <= dt < dstoff
else:
isdst = not dstoff <= dt < dston
return isdst
@property
def _dst_base_offset(self):
return self._dst_offset - self._std_offset
__hash__ = None
def __ne__(self, other):
return not (self == other)
def __repr__(self):
return "%s(...)" % self.__class__.__name__
__reduce__ = object.__reduce__
+80
View File
@@ -0,0 +1,80 @@
from datetime import timedelta
import weakref
from collections import OrderedDict
from six.moves import _thread
class _TzSingleton(type):
def __init__(cls, *args, **kwargs):
cls.__instance = None
super(_TzSingleton, cls).__init__(*args, **kwargs)
def __call__(cls):
if cls.__instance is None:
cls.__instance = super(_TzSingleton, cls).__call__()
return cls.__instance
class _TzFactory(type):
def instance(cls, *args, **kwargs):
"""Alternate constructor that returns a fresh instance"""
return type.__call__(cls, *args, **kwargs)
class _TzOffsetFactory(_TzFactory):
def __init__(cls, *args, **kwargs):
cls.__instances = weakref.WeakValueDictionary()
cls.__strong_cache = OrderedDict()
cls.__strong_cache_size = 8
cls._cache_lock = _thread.allocate_lock()
def __call__(cls, name, offset):
if isinstance(offset, timedelta):
key = (name, offset.total_seconds())
else:
key = (name, offset)
instance = cls.__instances.get(key, None)
if instance is None:
instance = cls.__instances.setdefault(key,
cls.instance(name, offset))
# This lock may not be necessary in Python 3. See GH issue #901
with cls._cache_lock:
cls.__strong_cache[key] = cls.__strong_cache.pop(key, instance)
# Remove an item if the strong cache is overpopulated
if len(cls.__strong_cache) > cls.__strong_cache_size:
cls.__strong_cache.popitem(last=False)
return instance
class _TzStrFactory(_TzFactory):
def __init__(cls, *args, **kwargs):
cls.__instances = weakref.WeakValueDictionary()
cls.__strong_cache = OrderedDict()
cls.__strong_cache_size = 8
cls.__cache_lock = _thread.allocate_lock()
def __call__(cls, s, posix_offset=False):
key = (s, posix_offset)
instance = cls.__instances.get(key, None)
if instance is None:
instance = cls.__instances.setdefault(key,
cls.instance(s, posix_offset))
# This lock may not be necessary in Python 3. See GH issue #901
with cls.__cache_lock:
cls.__strong_cache[key] = cls.__strong_cache.pop(key, instance)
# Remove an item if the strong cache is overpopulated
if len(cls.__strong_cache) > cls.__strong_cache_size:
cls.__strong_cache.popitem(last=False)
return instance
+1849
View File
File diff suppressed because it is too large Load Diff
+370
View File
@@ -0,0 +1,370 @@
# -*- coding: utf-8 -*-
"""
This module provides an interface to the native time zone data on Windows,
including :py:class:`datetime.tzinfo` implementations.
Attempting to import this module on a non-Windows platform will raise an
:py:obj:`ImportError`.
"""
# This code was originally contributed by Jeffrey Harris.
import datetime
import struct
from six.moves import winreg
from six import text_type
try:
import ctypes
from ctypes import wintypes
except ValueError:
# ValueError is raised on non-Windows systems for some horrible reason.
raise ImportError("Running tzwin on non-Windows system")
from ._common import tzrangebase
__all__ = ["tzwin", "tzwinlocal", "tzres"]
ONEWEEK = datetime.timedelta(7)
TZKEYNAMENT = r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Time Zones"
TZKEYNAME9X = r"SOFTWARE\Microsoft\Windows\CurrentVersion\Time Zones"
TZLOCALKEYNAME = r"SYSTEM\CurrentControlSet\Control\TimeZoneInformation"
def _settzkeyname():
handle = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE)
try:
winreg.OpenKey(handle, TZKEYNAMENT).Close()
TZKEYNAME = TZKEYNAMENT
except WindowsError:
TZKEYNAME = TZKEYNAME9X
handle.Close()
return TZKEYNAME
TZKEYNAME = _settzkeyname()
class tzres(object):
"""
Class for accessing ``tzres.dll``, which contains timezone name related
resources.
.. versionadded:: 2.5.0
"""
p_wchar = ctypes.POINTER(wintypes.WCHAR) # Pointer to a wide char
def __init__(self, tzres_loc='tzres.dll'):
# Load the user32 DLL so we can load strings from tzres
user32 = ctypes.WinDLL('user32')
# Specify the LoadStringW function
user32.LoadStringW.argtypes = (wintypes.HINSTANCE,
wintypes.UINT,
wintypes.LPWSTR,
ctypes.c_int)
self.LoadStringW = user32.LoadStringW
self._tzres = ctypes.WinDLL(tzres_loc)
self.tzres_loc = tzres_loc
def load_name(self, offset):
"""
Load a timezone name from a DLL offset (integer).
>>> from dateutil.tzwin import tzres
>>> tzr = tzres()
>>> print(tzr.load_name(112))
'Eastern Standard Time'
:param offset:
A positive integer value referring to a string from the tzres dll.
.. note::
Offsets found in the registry are generally of the form
``@tzres.dll,-114``. The offset in this case is 114, not -114.
"""
resource = self.p_wchar()
lpBuffer = ctypes.cast(ctypes.byref(resource), wintypes.LPWSTR)
nchar = self.LoadStringW(self._tzres._handle, offset, lpBuffer, 0)
return resource[:nchar]
def name_from_string(self, tzname_str):
"""
Parse strings as returned from the Windows registry into the time zone
name as defined in the registry.
>>> from dateutil.tzwin import tzres
>>> tzr = tzres()
>>> print(tzr.name_from_string('@tzres.dll,-251'))
'Dateline Daylight Time'
>>> print(tzr.name_from_string('Eastern Standard Time'))
'Eastern Standard Time'
:param tzname_str:
A timezone name string as returned from a Windows registry key.
:return:
Returns the localized timezone string from tzres.dll if the string
is of the form `@tzres.dll,-offset`, else returns the input string.
"""
if not tzname_str.startswith('@'):
return tzname_str
name_splt = tzname_str.split(',-')
try:
offset = int(name_splt[1])
except:
raise ValueError("Malformed timezone string.")
return self.load_name(offset)
class tzwinbase(tzrangebase):
"""tzinfo class based on win32's timezones available in the registry."""
def __init__(self):
raise NotImplementedError('tzwinbase is an abstract base class')
def __eq__(self, other):
# Compare on all relevant dimensions, including name.
if not isinstance(other, tzwinbase):
return NotImplemented
return (self._std_offset == other._std_offset and
self._dst_offset == other._dst_offset and
self._stddayofweek == other._stddayofweek and
self._dstdayofweek == other._dstdayofweek and
self._stdweeknumber == other._stdweeknumber and
self._dstweeknumber == other._dstweeknumber and
self._stdhour == other._stdhour and
self._dsthour == other._dsthour and
self._stdminute == other._stdminute and
self._dstminute == other._dstminute and
self._std_abbr == other._std_abbr and
self._dst_abbr == other._dst_abbr)
@staticmethod
def list():
"""Return a list of all time zones known to the system."""
with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
with winreg.OpenKey(handle, TZKEYNAME) as tzkey:
result = [winreg.EnumKey(tzkey, i)
for i in range(winreg.QueryInfoKey(tzkey)[0])]
return result
def display(self):
"""
Return the display name of the time zone.
"""
return self._display
def transitions(self, year):
"""
For a given year, get the DST on and off transition times, expressed
always on the standard time side. For zones with no transitions, this
function returns ``None``.
:param year:
The year whose transitions you would like to query.
:return:
Returns a :class:`tuple` of :class:`datetime.datetime` objects,
``(dston, dstoff)`` for zones with an annual DST transition, or
``None`` for fixed offset zones.
"""
if not self.hasdst:
return None
dston = picknthweekday(year, self._dstmonth, self._dstdayofweek,
self._dsthour, self._dstminute,
self._dstweeknumber)
dstoff = picknthweekday(year, self._stdmonth, self._stddayofweek,
self._stdhour, self._stdminute,
self._stdweeknumber)
# Ambiguous dates default to the STD side
dstoff -= self._dst_base_offset
return dston, dstoff
def _get_hasdst(self):
return self._dstmonth != 0
@property
def _dst_base_offset(self):
return self._dst_base_offset_
class tzwin(tzwinbase):
"""
Time zone object created from the zone info in the Windows registry
These are similar to :py:class:`dateutil.tz.tzrange` objects in that
the time zone data is provided in the format of a single offset rule
for either 0 or 2 time zone transitions per year.
:param: name
The name of a Windows time zone key, e.g. "Eastern Standard Time".
The full list of keys can be retrieved with :func:`tzwin.list`.
"""
def __init__(self, name):
self._name = name
with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
tzkeyname = text_type("{kn}\\{name}").format(kn=TZKEYNAME, name=name)
with winreg.OpenKey(handle, tzkeyname) as tzkey:
keydict = valuestodict(tzkey)
self._std_abbr = keydict["Std"]
self._dst_abbr = keydict["Dlt"]
self._display = keydict["Display"]
# See http://ww_winreg.jsiinc.com/SUBA/tip0300/rh0398.htm
tup = struct.unpack("=3l16h", keydict["TZI"])
stdoffset = -tup[0]-tup[1] # Bias + StandardBias * -1
dstoffset = stdoffset-tup[2] # + DaylightBias * -1
self._std_offset = datetime.timedelta(minutes=stdoffset)
self._dst_offset = datetime.timedelta(minutes=dstoffset)
# for the meaning see the win32 TIME_ZONE_INFORMATION structure docs
# http://msdn.microsoft.com/en-us/library/windows/desktop/ms725481(v=vs.85).aspx
(self._stdmonth,
self._stddayofweek, # Sunday = 0
self._stdweeknumber, # Last = 5
self._stdhour,
self._stdminute) = tup[4:9]
(self._dstmonth,
self._dstdayofweek, # Sunday = 0
self._dstweeknumber, # Last = 5
self._dsthour,
self._dstminute) = tup[12:17]
self._dst_base_offset_ = self._dst_offset - self._std_offset
self.hasdst = self._get_hasdst()
def __repr__(self):
return "tzwin(%s)" % repr(self._name)
def __reduce__(self):
return (self.__class__, (self._name,))
class tzwinlocal(tzwinbase):
"""
Class representing the local time zone information in the Windows registry
While :class:`dateutil.tz.tzlocal` makes system calls (via the :mod:`time`
module) to retrieve time zone information, ``tzwinlocal`` retrieves the
rules directly from the Windows registry and creates an object like
:class:`dateutil.tz.tzwin`.
Because Windows does not have an equivalent of :func:`time.tzset`, on
Windows, :class:`dateutil.tz.tzlocal` instances will always reflect the
time zone settings *at the time that the process was started*, meaning
changes to the machine's time zone settings during the run of a program
on Windows will **not** be reflected by :class:`dateutil.tz.tzlocal`.
Because ``tzwinlocal`` reads the registry directly, it is unaffected by
this issue.
"""
def __init__(self):
with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
with winreg.OpenKey(handle, TZLOCALKEYNAME) as tzlocalkey:
keydict = valuestodict(tzlocalkey)
self._std_abbr = keydict["StandardName"]
self._dst_abbr = keydict["DaylightName"]
try:
tzkeyname = text_type('{kn}\\{sn}').format(kn=TZKEYNAME,
sn=self._std_abbr)
with winreg.OpenKey(handle, tzkeyname) as tzkey:
_keydict = valuestodict(tzkey)
self._display = _keydict["Display"]
except OSError:
self._display = None
stdoffset = -keydict["Bias"]-keydict["StandardBias"]
dstoffset = stdoffset-keydict["DaylightBias"]
self._std_offset = datetime.timedelta(minutes=stdoffset)
self._dst_offset = datetime.timedelta(minutes=dstoffset)
# For reasons unclear, in this particular key, the day of week has been
# moved to the END of the SYSTEMTIME structure.
tup = struct.unpack("=8h", keydict["StandardStart"])
(self._stdmonth,
self._stdweeknumber, # Last = 5
self._stdhour,
self._stdminute) = tup[1:5]
self._stddayofweek = tup[7]
tup = struct.unpack("=8h", keydict["DaylightStart"])
(self._dstmonth,
self._dstweeknumber, # Last = 5
self._dsthour,
self._dstminute) = tup[1:5]
self._dstdayofweek = tup[7]
self._dst_base_offset_ = self._dst_offset - self._std_offset
self.hasdst = self._get_hasdst()
def __repr__(self):
return "tzwinlocal()"
def __str__(self):
# str will return the standard name, not the daylight name.
return "tzwinlocal(%s)" % repr(self._std_abbr)
def __reduce__(self):
return (self.__class__, ())
def picknthweekday(year, month, dayofweek, hour, minute, whichweek):
""" dayofweek == 0 means Sunday, whichweek 5 means last instance """
first = datetime.datetime(year, month, 1, hour, minute)
# This will work if dayofweek is ISO weekday (1-7) or Microsoft-style (0-6),
# Because 7 % 7 = 0
weekdayone = first.replace(day=((dayofweek - first.isoweekday()) % 7) + 1)
wd = weekdayone + ((whichweek - 1) * ONEWEEK)
if (wd.month != month):
wd -= ONEWEEK
return wd
def valuestodict(key):
"""Convert a registry key's values to a dictionary."""
dout = {}
size = winreg.QueryInfoKey(key)[1]
tz_res = None
for i in range(size):
key_name, value, dtype = winreg.EnumValue(key, i)
if dtype == winreg.REG_DWORD or dtype == winreg.REG_DWORD_LITTLE_ENDIAN:
# If it's a DWORD (32-bit integer), it's stored as unsigned - convert
# that to a proper signed integer
if value & (1 << 31):
value = value - (1 << 32)
elif dtype == winreg.REG_SZ:
# If it's a reference to the tzres DLL, load the actual string
if value.startswith('@tzres'):
tz_res = tz_res or tzres()
value = tz_res.name_from_string(value)
value = value.rstrip('\x00') # Remove trailing nulls
dout[key_name] = value
return dout
+2
View File
@@ -0,0 +1,2 @@
# tzwin has moved to dateutil.tz.win
from .tz.win import *
+71
View File
@@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
"""
This module offers general convenience and utility functions for dealing with
datetimes.
.. versionadded:: 2.7.0
"""
from __future__ import unicode_literals
from datetime import datetime, time
def today(tzinfo=None):
"""
Returns a :py:class:`datetime` representing the current day at midnight
:param tzinfo:
The time zone to attach (also used to determine the current day).
:return:
A :py:class:`datetime.datetime` object representing the current day
at midnight.
"""
dt = datetime.now(tzinfo)
return datetime.combine(dt.date(), time(0, tzinfo=tzinfo))
def default_tzinfo(dt, tzinfo):
"""
Sets the ``tzinfo`` parameter on naive datetimes only
This is useful for example when you are provided a datetime that may have
either an implicit or explicit time zone, such as when parsing a time zone
string.
.. doctest::
>>> from dateutil.tz import tzoffset
>>> from dateutil.parser import parse
>>> from dateutil.utils import default_tzinfo
>>> dflt_tz = tzoffset("EST", -18000)
>>> print(default_tzinfo(parse('2014-01-01 12:30 UTC'), dflt_tz))
2014-01-01 12:30:00+00:00
>>> print(default_tzinfo(parse('2014-01-01 12:30'), dflt_tz))
2014-01-01 12:30:00-05:00
:param dt:
The datetime on which to replace the time zone
:param tzinfo:
The :py:class:`datetime.tzinfo` subclass instance to assign to
``dt`` if (and only if) it is naive.
:return:
Returns an aware :py:class:`datetime.datetime`.
"""
if dt.tzinfo is not None:
return dt
else:
return dt.replace(tzinfo=tzinfo)
def within_delta(dt1, dt2, delta):
"""
Useful for comparing two datetimes that may a negilible difference
to be considered equal.
"""
delta = abs(delta)
difference = dt1 - dt2
return -delta <= difference <= delta
+167
View File
@@ -0,0 +1,167 @@
# -*- coding: utf-8 -*-
import warnings
import json
from tarfile import TarFile
from pkgutil import get_data
from io import BytesIO
from dateutil.tz import tzfile as _tzfile
__all__ = ["get_zonefile_instance", "gettz", "gettz_db_metadata"]
ZONEFILENAME = "dateutil-zoneinfo.tar.gz"
METADATA_FN = 'METADATA'
class tzfile(_tzfile):
def __reduce__(self):
return (gettz, (self._filename,))
def getzoneinfofile_stream():
try:
return BytesIO(get_data(__name__, ZONEFILENAME))
except IOError as e: # TODO switch to FileNotFoundError?
warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
return None
class ZoneInfoFile(object):
def __init__(self, zonefile_stream=None):
if zonefile_stream is not None:
with TarFile.open(fileobj=zonefile_stream) as tf:
self.zones = {zf.name: tzfile(tf.extractfile(zf), filename=zf.name)
for zf in tf.getmembers()
if zf.isfile() and zf.name != METADATA_FN}
# deal with links: They'll point to their parent object. Less
# waste of memory
links = {zl.name: self.zones[zl.linkname]
for zl in tf.getmembers() if
zl.islnk() or zl.issym()}
self.zones.update(links)
try:
metadata_json = tf.extractfile(tf.getmember(METADATA_FN))
metadata_str = metadata_json.read().decode('UTF-8')
self.metadata = json.loads(metadata_str)
except KeyError:
# no metadata in tar file
self.metadata = None
else:
self.zones = {}
self.metadata = None
def get(self, name, default=None):
"""
Wrapper for :func:`ZoneInfoFile.zones.get`. This is a convenience method
for retrieving zones from the zone dictionary.
:param name:
The name of the zone to retrieve. (Generally IANA zone names)
:param default:
The value to return in the event of a missing key.
.. versionadded:: 2.6.0
"""
return self.zones.get(name, default)
# The current API has gettz as a module function, although in fact it taps into
# a stateful class. So as a workaround for now, without changing the API, we
# will create a new "global" class instance the first time a user requests a
# timezone. Ugly, but adheres to the api.
#
# TODO: Remove after deprecation period.
_CLASS_ZONE_INSTANCE = []
def get_zonefile_instance(new_instance=False):
"""
This is a convenience function which provides a :class:`ZoneInfoFile`
instance using the data provided by the ``dateutil`` package. By default, it
caches a single instance of the ZoneInfoFile object and returns that.
:param new_instance:
If ``True``, a new instance of :class:`ZoneInfoFile` is instantiated and
used as the cached instance for the next call. Otherwise, new instances
are created only as necessary.
:return:
Returns a :class:`ZoneInfoFile` object.
.. versionadded:: 2.6
"""
if new_instance:
zif = None
else:
zif = getattr(get_zonefile_instance, '_cached_instance', None)
if zif is None:
zif = ZoneInfoFile(getzoneinfofile_stream())
get_zonefile_instance._cached_instance = zif
return zif
def gettz(name):
"""
This retrieves a time zone from the local zoneinfo tarball that is packaged
with dateutil.
:param name:
An IANA-style time zone name, as found in the zoneinfo file.
:return:
Returns a :class:`dateutil.tz.tzfile` time zone object.
.. warning::
It is generally inadvisable to use this function, and it is only
provided for API compatibility with earlier versions. This is *not*
equivalent to ``dateutil.tz.gettz()``, which selects an appropriate
time zone based on the inputs, favoring system zoneinfo. This is ONLY
for accessing the dateutil-specific zoneinfo (which may be out of
date compared to the system zoneinfo).
.. deprecated:: 2.6
If you need to use a specific zoneinfofile over the system zoneinfo,
instantiate a :class:`dateutil.zoneinfo.ZoneInfoFile` object and call
:func:`dateutil.zoneinfo.ZoneInfoFile.get(name)` instead.
Use :func:`get_zonefile_instance` to retrieve an instance of the
dateutil-provided zoneinfo.
"""
warnings.warn("zoneinfo.gettz() will be removed in future versions, "
"to use the dateutil-provided zoneinfo files, instantiate a "
"ZoneInfoFile object and use ZoneInfoFile.zones.get() "
"instead. See the documentation for details.",
DeprecationWarning)
if len(_CLASS_ZONE_INSTANCE) == 0:
_CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
return _CLASS_ZONE_INSTANCE[0].zones.get(name)
def gettz_db_metadata():
""" Get the zonefile metadata
See `zonefile_metadata`_
:returns:
A dictionary with the database metadata
.. deprecated:: 2.6
See deprecation warning in :func:`zoneinfo.gettz`. To get metadata,
query the attribute ``zoneinfo.ZoneInfoFile.metadata``.
"""
warnings.warn("zoneinfo.gettz_db_metadata() will be removed in future "
"versions, to use the dateutil-provided zoneinfo files, "
"ZoneInfoFile object and query the 'metadata' attribute "
"instead. See the documentation for details.",
DeprecationWarning)
if len(_CLASS_ZONE_INSTANCE) == 0:
_CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
return _CLASS_ZONE_INSTANCE[0].metadata
Binary file not shown.
+53
View File
@@ -0,0 +1,53 @@
import logging
import os
import tempfile
import shutil
import json
from subprocess import check_call
from tarfile import TarFile
from dateutil.zoneinfo import METADATA_FN, ZONEFILENAME
def rebuild(filename, tag=None, format="gz", zonegroups=[], metadata=None):
"""Rebuild the internal timezone info in dateutil/zoneinfo/zoneinfo*tar*
filename is the timezone tarball from ``ftp.iana.org/tz``.
"""
tmpdir = tempfile.mkdtemp()
zonedir = os.path.join(tmpdir, "zoneinfo")
moduledir = os.path.dirname(__file__)
try:
with TarFile.open(filename) as tf:
for name in zonegroups:
tf.extract(name, tmpdir)
filepaths = [os.path.join(tmpdir, n) for n in zonegroups]
try:
check_call(["zic", "-d", zonedir] + filepaths)
except OSError as e:
_print_on_nosuchfile(e)
raise
# write metadata file
with open(os.path.join(zonedir, METADATA_FN), 'w') as f:
json.dump(metadata, f, indent=4, sort_keys=True)
target = os.path.join(moduledir, ZONEFILENAME)
with TarFile.open(target, "w:%s" % format) as tf:
for entry in os.listdir(zonedir):
entrypath = os.path.join(zonedir, entry)
tf.add(entrypath, entry)
finally:
shutil.rmtree(tmpdir)
def _print_on_nosuchfile(e):
"""Print helpful troubleshooting message
e is an exception raised by subprocess.check_call()
"""
if e.errno == 2:
logging.error(
"Could not find zic. Perhaps you need to install "
"libc-bin or some other package that provides it, "
"or it's not in your PATH?")
+13
View File
@@ -47,6 +47,19 @@ except ImportError:
from zipp import Path as ZipPath # type: ignore
try:
from typing import runtime_checkable # type: ignore
except ImportError:
def runtime_checkable(cls): # type: ignore
return cls
try:
from typing import Protocol # type: ignore
except ImportError:
Protocol = ABC # type: ignore
class PackageSpec(object):
def __init__(self, **kwargs):
vars(self).update(kwargs)
+10 -2
View File
@@ -2,7 +2,7 @@ from __future__ import absolute_import
import abc
from ._compat import ABC, FileNotFoundError
from ._compat import ABC, FileNotFoundError, runtime_checkable, Protocol
# Use mypy's comment syntax for Python 2 compatibility
try:
@@ -57,7 +57,8 @@ class ResourceReader(ABC):
raise FileNotFoundError
class Traversable(ABC):
@runtime_checkable
class Traversable(Protocol):
"""
An object with a subset of pathlib.Path methods suitable for
traversing directories and opening files.
@@ -115,6 +116,13 @@ class Traversable(ABC):
accepted by io.TextIOWrapper.
"""
@abc.abstractproperty
def name(self):
# type: () -> str
"""
The base name of this object without any parent references.
"""
class TraversableResources(ResourceReader):
@abc.abstractmethod
+39
View File
@@ -0,0 +1,39 @@
import typing
import unittest
import importlib_resources as resources
from importlib_resources.abc import Traversable
from . import data01
from . import util
class FilesTests:
def test_read_bytes(self):
files = resources.files(self.data)
actual = files.joinpath('utf-8.file').read_bytes()
assert actual == b'Hello, UTF-8 world!\n'
def test_read_text(self):
files = resources.files(self.data)
actual = files.joinpath('utf-8.file').read_text()
assert actual == 'Hello, UTF-8 world!\n'
@unittest.skipUnless(
hasattr(typing, 'runtime_checkable'),
"Only suitable when typing supports runtime_checkable",
)
def test_traversable(self):
assert isinstance(resources.files(self.data), Traversable)
class OpenDiskTests(FilesTests, unittest.TestCase):
def setUp(self):
self.data = data01
class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase):
pass
if __name__ == '__main__':
unittest.main()
+1 -1
View File
@@ -41,4 +41,4 @@ from .utils import evalcontextfunction
from .utils import is_undefined
from .utils import select_autoescape
__version__ = "2.11.1"
__version__ = "2.11.2"
+4 -5
View File
@@ -26,17 +26,16 @@ async def async_select_or_reject(args, kwargs, modfunc, lookup_attr):
def dualfilter(normal_filter, async_filter):
wrap_evalctx = False
if getattr(normal_filter, "environmentfilter", False):
if getattr(normal_filter, "environmentfilter", False) is True:
def is_async(args):
return args[0].is_async
wrap_evalctx = False
else:
if not getattr(normal_filter, "evalcontextfilter", False) and not getattr(
normal_filter, "contextfilter", False
):
wrap_evalctx = True
has_evalctxfilter = getattr(normal_filter, "evalcontextfilter", False) is True
has_ctxfilter = getattr(normal_filter, "contextfilter", False) is True
wrap_evalctx = not has_evalctxfilter and not has_ctxfilter
def is_async(args):
return args[0].environment.is_async
+6 -6
View File
@@ -1307,13 +1307,13 @@ class CodeGenerator(NodeVisitor):
def finalize(value):
return default(env_finalize(value))
if getattr(env_finalize, "contextfunction", False):
if getattr(env_finalize, "contextfunction", False) is True:
src += "context, "
finalize = None # noqa: F811
elif getattr(env_finalize, "evalcontextfunction", False):
elif getattr(env_finalize, "evalcontextfunction", False) is True:
src += "context.eval_ctx, "
finalize = None
elif getattr(env_finalize, "environmentfunction", False):
elif getattr(env_finalize, "environmentfunction", False) is True:
src += "environment, "
def finalize(value):
@@ -1689,11 +1689,11 @@ class CodeGenerator(NodeVisitor):
func = self.environment.filters.get(node.name)
if func is None:
self.fail("no filter named %r" % node.name, node.lineno)
if getattr(func, "contextfilter", False):
if getattr(func, "contextfilter", False) is True:
self.write("context, ")
elif getattr(func, "evalcontextfilter", False):
elif getattr(func, "evalcontextfilter", False) is True:
self.write("context.eval_ctx, ")
elif getattr(func, "environmentfilter", False):
elif getattr(func, "environmentfilter", False) is True:
self.write("environment, ")
# if the filter node is None we are inside a filter block
+1 -4
View File
@@ -245,10 +245,7 @@ else:
class _CTraceback(ctypes.Structure):
_fields_ = [
# Extra PyObject slots when compiled with Py_TRACE_REFS.
(
"PyObject_HEAD",
ctypes.c_byte * (32 if hasattr(sys, "getobjects") else 16),
),
("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()),
# Only care about tb_next as an object, not a traceback.
("tb_next", ctypes.py_object),
]
+3 -3
View File
@@ -492,20 +492,20 @@ class Environment(object):
if func is None:
fail_for_missing_callable("no filter named %r", name)
args = [value] + list(args or ())
if getattr(func, "contextfilter", False):
if getattr(func, "contextfilter", False) is True:
if context is None:
raise TemplateRuntimeError(
"Attempted to invoke context filter without context"
)
args.insert(0, context)
elif getattr(func, "evalcontextfilter", False):
elif getattr(func, "evalcontextfilter", False) is True:
if eval_ctx is None:
if context is not None:
eval_ctx = context.eval_ctx
else:
eval_ctx = EvalContext(self)
args.insert(0, eval_ctx)
elif getattr(func, "environmentfilter", False):
elif getattr(func, "environmentfilter", False) is True:
args.insert(0, self)
return func(*args, **(kwargs or {}))
+1 -1
View File
@@ -761,7 +761,7 @@ def do_wordwrap(
def do_wordcount(s):
"""Count the words in that string."""
return len(_word_re.findall(s))
return len(_word_re.findall(soft_unicode(s)))
def do_int(value, default=0, base=10):
+14 -7
View File
@@ -681,6 +681,8 @@ class Lexer(object):
source_length = len(source)
balancing_stack = []
lstrip_unless_re = self.lstrip_unless_re
newlines_stripped = 0
line_starting = True
while 1:
# tokenizer loop
@@ -717,7 +719,9 @@ class Lexer(object):
if strip_sign == "-":
# Strip all whitespace between the text and the tag.
groups = (text.rstrip(),) + groups[1:]
stripped = text.rstrip()
newlines_stripped = text[len(stripped) :].count("\n")
groups = (stripped,) + groups[1:]
elif (
# Not marked for preserving whitespace.
strip_sign != "+"
@@ -728,11 +732,11 @@ class Lexer(object):
):
# The start of text between the last newline and the tag.
l_pos = text.rfind("\n") + 1
# If there's only whitespace between the newline and the
# tag, strip it.
if not lstrip_unless_re.search(text, l_pos):
groups = (text[:l_pos],) + groups[1:]
if l_pos > 0 or line_starting:
# If there's only whitespace between the newline and the
# tag, strip it.
if not lstrip_unless_re.search(text, l_pos):
groups = (text[:l_pos],) + groups[1:]
for idx, token in enumerate(tokens):
# failure group
@@ -758,7 +762,8 @@ class Lexer(object):
data = groups[idx]
if data or token not in ignore_if_empty:
yield lineno, token, data
lineno += data.count("\n")
lineno += data.count("\n") + newlines_stripped
newlines_stripped = 0
# strings as token just are yielded as it.
else:
@@ -790,6 +795,8 @@ class Lexer(object):
yield lineno, tokens, data
lineno += data.count("\n")
line_starting = m.group()[-1:] == "\n"
# fetch new position into new variable so that we can check
# if there is a internal parsing error which would result
# in an infinite loop
+48 -116
View File
@@ -3,11 +3,9 @@
sources.
"""
import os
import pkgutil
import sys
import weakref
from hashlib import sha1
from importlib import import_module
from os import path
from types import ModuleType
@@ -217,141 +215,75 @@ class FileSystemLoader(BaseLoader):
class PackageLoader(BaseLoader):
"""Load templates from a directory in a Python package.
"""Load templates from python eggs or packages. It is constructed with
the name of the python package and the path to the templates in that
package::
:param package_name: Import name of the package that contains the
template directory.
:param package_path: Directory within the imported package that
contains the templates.
:param encoding: Encoding of template files.
loader = PackageLoader('mypackage', 'views')
The following example looks up templates in the ``pages`` directory
within the ``project.ui`` package.
If the package path is not given, ``'templates'`` is assumed.
.. code-block:: python
loader = PackageLoader("project.ui", "pages")
Only packages installed as directories (standard pip behavior) or
zip/egg files (less common) are supported. The Python API for
introspecting data in packages is too limited to support other
installation methods the way this loader requires.
There is limited support for :pep:`420` namespace packages. The
template directory is assumed to only be in one namespace
contributor. Zip files contributing to a namespace are not
supported.
.. versionchanged:: 2.11.0
No longer uses ``setuptools`` as a dependency.
.. versionchanged:: 2.11.0
Limited PEP 420 namespace package support.
Per default the template encoding is ``'utf-8'`` which can be changed
by setting the `encoding` parameter to something else. Due to the nature
of eggs it's only possible to reload templates if the package was loaded
from the file system and not a zip file.
"""
def __init__(self, package_name, package_path="templates", encoding="utf-8"):
if package_path == os.path.curdir:
package_path = ""
elif package_path[:2] == os.path.curdir + os.path.sep:
package_path = package_path[2:]
from pkg_resources import DefaultProvider
from pkg_resources import get_provider
from pkg_resources import ResourceManager
package_path = os.path.normpath(package_path).rstrip(os.path.sep)
self.package_path = package_path
self.package_name = package_name
provider = get_provider(package_name)
self.encoding = encoding
# Make sure the package exists. This also makes namespace
# packages work, otherwise get_loader returns None.
import_module(package_name)
self._loader = loader = pkgutil.get_loader(package_name)
# Zip loader's archive attribute points at the zip.
self._archive = getattr(loader, "archive", None)
self._template_root = None
if hasattr(loader, "get_filename"):
# A standard directory package, or a zip package.
self._template_root = os.path.join(
os.path.dirname(loader.get_filename(package_name)), package_path
)
elif hasattr(loader, "_path"):
# A namespace package, limited support. Find the first
# contributor with the template directory.
for root in loader._path:
root = os.path.join(root, package_path)
if os.path.isdir(root):
self._template_root = root
break
if self._template_root is None:
raise ValueError(
"The %r package was not installed in a way that"
" PackageLoader understands." % package_name
)
self.manager = ResourceManager()
self.filesystem_bound = isinstance(provider, DefaultProvider)
self.provider = provider
self.package_path = package_path
def get_source(self, environment, template):
p = os.path.join(self._template_root, *split_template_path(template))
pieces = split_template_path(template)
p = "/".join((self.package_path,) + tuple(pieces))
if self._archive is None:
# Package is a directory.
if not os.path.isfile(p):
raise TemplateNotFound(template)
if not self.provider.has_resource(p):
raise TemplateNotFound(template)
with open(p, "rb") as f:
source = f.read()
filename = uptodate = None
mtime = os.path.getmtime(p)
if self.filesystem_bound:
filename = self.provider.get_resource_filename(self.manager, p)
mtime = path.getmtime(filename)
def up_to_date():
return os.path.isfile(p) and os.path.getmtime(p) == mtime
def uptodate():
try:
return path.getmtime(filename) == mtime
except OSError:
return False
else:
# Package is a zip file.
try:
source = self._loader.get_data(p)
except OSError:
raise TemplateNotFound(template)
# Could use the zip's mtime for all template mtimes, but
# would need to safely reload the module if it's out of
# date, so just report it as always current.
up_to_date = None
return source.decode(self.encoding), p, up_to_date
source = self.provider.get_resource_string(self.manager, p)
return source.decode(self.encoding), filename, uptodate
def list_templates(self):
path = self.package_path
if path[:2] == "./":
path = path[2:]
elif path == ".":
path = ""
offset = len(path)
results = []
if self._archive is None:
# Package is a directory.
offset = len(self._template_root)
def _walk(path):
for filename in self.provider.resource_listdir(path):
fullname = path + "/" + filename
for dirpath, _, filenames in os.walk(self._template_root):
dirpath = dirpath[offset:].lstrip(os.path.sep)
results.extend(
os.path.join(dirpath, name).replace(os.path.sep, "/")
for name in filenames
)
else:
if not hasattr(self._loader, "_files"):
raise TypeError(
"This zip import does not have the required"
" metadata to list templates."
)
# Package is a zip file.
prefix = (
self._template_root[len(self._archive) :].lstrip(os.path.sep)
+ os.path.sep
)
offset = len(prefix)
for name in self._loader._files.keys():
# Find names under the templates directory that aren't directories.
if name.startswith(prefix) and name[-1] != os.path.sep:
results.append(name[offset:].replace(os.path.sep, "/"))
if self.provider.resource_isdir(fullname):
_walk(fullname)
else:
results.append(fullname[offset:].lstrip("/"))
_walk(path)
results.sort()
return results
+7 -24
View File
@@ -1,4 +1,3 @@
import types
from ast import literal_eval
from itertools import chain
from itertools import islice
@@ -11,7 +10,7 @@ from .environment import Environment
from .environment import Template
def native_concat(nodes, preserve_quotes=True):
def native_concat(nodes):
"""Return a native Python type from the list of compiled nodes. If
the result is a single node, its value is returned. Otherwise, the
nodes are concatenated as strings. If the result can be parsed with
@@ -19,9 +18,6 @@ def native_concat(nodes, preserve_quotes=True):
the string is returned.
:param nodes: Iterable of nodes to concatenate.
:param preserve_quotes: Whether to re-wrap literal strings with
quotes, to preserve quotes around expressions for later parsing.
Should be ``False`` in :meth:`NativeEnvironment.render`.
"""
head = list(islice(nodes, 2))
@@ -31,29 +27,17 @@ def native_concat(nodes, preserve_quotes=True):
if len(head) == 1:
raw = head[0]
else:
if isinstance(nodes, types.GeneratorType):
nodes = chain(head, nodes)
raw = u"".join([text_type(v) for v in nodes])
raw = u"".join([text_type(v) for v in chain(head, nodes)])
try:
literal = literal_eval(raw)
return literal_eval(raw)
except (ValueError, SyntaxError, MemoryError):
return raw
# If literal_eval returned a string, re-wrap with the original
# quote character to avoid dropping quotes between expression nodes.
# Without this, "'{{ a }}', '{{ b }}'" results in "a, b", but should
# be ('a', 'b').
if preserve_quotes and isinstance(literal, str):
return "{quote}{}{quote}".format(literal, quote=raw[0])
return literal
class NativeCodeGenerator(CodeGenerator):
"""A code generator which renders Python types by not adding
``to_string()`` around output nodes, and using :func:`native_concat`
to convert complex strings back to Python types if possible.
``to_string()`` around output nodes.
"""
@staticmethod
@@ -61,7 +45,7 @@ class NativeCodeGenerator(CodeGenerator):
return value
def _output_const_repr(self, group):
return repr(native_concat(group))
return repr(u"".join([text_type(v) for v in group]))
def _output_child_to_const(self, node, frame, finalize):
const = node.as_const(frame.eval_ctx)
@@ -100,10 +84,9 @@ class NativeTemplate(Template):
Otherwise, the string is returned.
"""
vars = dict(*args, **kwargs)
try:
return native_concat(
self.root_render_func(self.new_context(vars)), preserve_quotes=False
)
return native_concat(self.root_render_func(self.new_context(vars)))
except Exception:
return self.environment.handle_exception()
+3 -3
View File
@@ -671,7 +671,7 @@ class Filter(Expr):
# python 3. because of that, do not rename filter_ to filter!
filter_ = self.environment.filters.get(self.name)
if filter_ is None or getattr(filter_, "contextfilter", False):
if filter_ is None or getattr(filter_, "contextfilter", False) is True:
raise Impossible()
# We cannot constant handle async filters, so we need to make sure
@@ -684,9 +684,9 @@ class Filter(Expr):
args, kwargs = args_as_const(self, eval_ctx)
args.insert(0, self.node.as_const(eval_ctx))
if getattr(filter_, "evalcontextfilter", False):
if getattr(filter_, "evalcontextfilter", False) is True:
args.insert(0, eval_ctx)
elif getattr(filter_, "environmentfilter", False):
elif getattr(filter_, "environmentfilter", False) is True:
args.insert(0, self.environment)
try:
+3 -3
View File
@@ -280,11 +280,11 @@ class Context(with_metaclass(ContextMeta)):
break
if callable(__obj):
if getattr(__obj, "contextfunction", 0):
if getattr(__obj, "contextfunction", False) is True:
args = (__self,) + args
elif getattr(__obj, "evalcontextfunction", 0):
elif getattr(__obj, "evalcontextfunction", False) is True:
args = (__self.eval_ctx,) + args
elif getattr(__obj, "environmentfunction", 0):
elif getattr(__obj, "environmentfunction", False) is True:
args = (__self.environment,) + args
try:
return __obj(*args, **kwargs)
+9 -4
View File
@@ -165,11 +165,15 @@ def object_type_repr(obj):
return "None"
elif obj is Ellipsis:
return "Ellipsis"
cls = type(obj)
# __builtin__ in 2.x, builtins in 3.x
if obj.__class__.__module__ in ("__builtin__", "builtins"):
name = obj.__class__.__name__
if cls.__module__ in ("__builtin__", "builtins"):
name = cls.__name__
else:
name = obj.__class__.__module__ + "." + obj.__class__.__name__
name = cls.__module__ + "." + cls.__name__
return "%s object" % name
@@ -693,7 +697,8 @@ class Namespace(object):
self.__attrs = dict(*args, **kwargs)
def __getattribute__(self, name):
if name == "_Namespace__attrs":
# __class__ is needed for the awaitable check in async mode
if name in {"_Namespace__attrs", "__class__"}:
return object.__getattribute__(self, name)
try:
return self.__attrs[name]
+1 -1
View File
@@ -10,7 +10,7 @@ from .models.lockfile import Lockfile
from .models.pipfile import Pipfile
from .models.requirements import Requirement
__version__ = "1.5.7"
__version__ = "1.5.9"
logger = logging.getLogger(__name__)
+24 -17
View File
@@ -57,6 +57,7 @@ if MYPY_RUNNING:
Command,
)
from packaging.requirements import Requirement as PackagingRequirement
from packaging.markers import Marker
TRequirement = TypeVar("TRequirement")
RequirementType = TypeVar(
@@ -71,9 +72,14 @@ PKGS_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "pkgs"))
WHEEL_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "wheels"))
DEPENDENCY_CACHE = DependencyCache()
WHEEL_CACHE = pip_shims.shims.WheelCache(
CACHE_DIR, pip_shims.shims.FormatControl(set(), set())
)
@contextlib.contextmanager
def _get_wheel_cache():
with pip_shims.shims.global_tempdir_manager():
yield pip_shims.shims.WheelCache(
CACHE_DIR, pip_shims.shims.FormatControl(set(), set())
)
def _get_filtered_versions(ireq, versions, prereleases):
@@ -351,6 +357,7 @@ def get_dependencies(ireq, sources=None, parent=None):
def get_dependencies_from_wheel_cache(ireq):
# type: (pip_shims.shims.InstallRequirement) -> Optional[Set[pip_shims.shims.InstallRequirement]]
"""Retrieves dependencies for the given install requirement from the wheel cache.
:param ireq: A single InstallRequirement
@@ -361,13 +368,14 @@ def get_dependencies_from_wheel_cache(ireq):
if ireq.editable or not is_pinned_requirement(ireq):
return
matches = WHEEL_CACHE.get(ireq.link, name_from_req(ireq.req))
if matches:
matches = set(matches)
if not DEPENDENCY_CACHE.get(ireq):
DEPENDENCY_CACHE[ireq] = [format_requirement(m) for m in matches]
return matches
return
with _get_wheel_cache() as wheel_cache:
matches = wheel_cache.get(ireq.link, name_from_req(ireq.req))
if matches:
matches = set(matches)
if not DEPENDENCY_CACHE.get(ireq):
DEPENDENCY_CACHE[ireq] = [format_requirement(m) for m in matches]
return matches
return None
def _marker_contains_extra(ireq):
@@ -477,12 +485,12 @@ def get_dependencies_from_index(dep, sources=None, pip_options=None, wheel_cache
"""
session, finder = get_finder(sources=sources, pip_options=pip_options)
if not wheel_cache:
wheel_cache = WHEEL_CACHE
dep.is_direct = True
requirements = None
setup_requires = {}
with temp_environ():
with temp_environ(), ExitStack() as stack:
if not wheel_cache:
wheel_cache = stack.enter_context(_get_wheel_cache())
os.environ["PIP_EXISTS_ACTION"] = "i"
if dep.editable and not dep.prepared and not dep.req:
setup_info = SetupInfo.from_ireq(dep)
@@ -570,10 +578,6 @@ def start_resolver(finder=None, session=None, wheel_cache=None):
if not session:
session = pip_command._build_session(pip_options)
if not wheel_cache:
wheel_cache = WHEEL_CACHE
_ensure_dir(fs_str(os.path.join(wheel_cache.cache_dir, "wheels")))
download_dir = PKGS_DOWNLOAD_DIR
_ensure_dir(download_dir)
@@ -582,6 +586,9 @@ def start_resolver(finder=None, session=None, wheel_cache=None):
try:
with ExitStack() as ctx:
ctx.enter_context(pip_shims.shims.global_tempdir_manager())
if not wheel_cache:
wheel_cache = ctx.enter_context(_get_wheel_cache())
_ensure_dir(fs_str(os.path.join(wheel_cache.cache_dir, "wheels")))
preparer = ctx.enter_context(
pip_shims.shims.make_preparer(
options=pip_options,
+2 -2
View File
@@ -25,7 +25,7 @@ if MYPY_RUNNING:
STRING_TYPE = Union[str, bytes, Text]
MAX_VERSIONS = {2: 7, 3: 11, 4: 0}
MAX_VERSIONS = {1: 7, 2: 7, 3: 11, 4: 0}
DEPRECATED_VERSIONS = ["3.0", "3.1", "3.2", "3.3"]
@@ -557,7 +557,7 @@ def _split_specifierset_str(specset_str, prefix="=="):
else:
values = [v.strip() for v in specset_str.split(",")]
if prefix == "!=" and any(v in values for v in DEPRECATED_VERSIONS):
values = DEPRECATED_VERSIONS[:]
values += DEPRECATED_VERSIONS[:]
for value in sorted(values):
specifiers.add(Specifier("{0}{1}".format(prefix, value)))
return specifiers
+6 -6
View File
@@ -443,8 +443,7 @@ class ParsedTag(object):
def parse_tag(tag):
# type: (Tag) -> ParsedTag
"""
Parse a :class:`~packaging.tags.Tag` instance
"""Parse a :class:`~packaging.tags.Tag` instance.
:param :class:`~packaging.tags.Tag` tag: A tag to parse
:return: A parsed tag with combined markers, supported platform and python version
@@ -520,6 +519,8 @@ class ReleaseUrl(object):
name = attr.ib(type=str, default=None)
#: The available comments of the given upload
comment_text = attr.ib(type=str, default="")
#: Whether the url has been yanked from the server
yanked = attr.ib(type=bool, default=False)
#: The number of downloads (deprecated)
downloads = attr.ib(type=int, default=-1)
#: The filename of the current upload
@@ -716,8 +717,8 @@ class ReleaseUrlCollection(Sequence):
def find_package_type(self, type_):
# type: (str) -> Optional[ReleaseUrl]
"""
Given a package type (e.g. sdist, bdist_wheel), find the matching release
"""Given a package type (e.g. sdist, bdist_wheel), find the matching
release.
:param str type_: A package type from :const:`~PACKAGE_TYPES`
:return: The package from this collection matching that type, if available
@@ -956,8 +957,7 @@ class PackageInfo(object):
def create_dependencies(self, force=False):
# type: (bool) -> "PackageInfo"
"""
Create values for **self.dependencies**.
"""Create values for **self.dependencies**.
:param bool force: Sets **self.dependencies** to an empty tuple if it would be
None, defaults to False.
+224 -165
View File
@@ -137,9 +137,6 @@ if MYPY_RUNNING:
SPECIFIERS_BY_LENGTH = sorted(list(Specifier._operators.keys()), key=len, reverse=True)
run = partial(vistir.misc.run, combine_stderr=False, return_object=True, nospin=True)
class Line(object):
def __init__(self, line, extras=None):
# type: (AnyStr, Optional[Union[List[S], Set[S], Tuple[S, ...]]]) -> None
@@ -164,8 +161,7 @@ class Line(object):
self.parsed_marker = None # type: Optional[Marker]
self.preferred_scheme = None # type: Optional[STRING_TYPE]
self._requirement = None # type: Optional[PackagingRequirement]
self.is_direct_url = False # type: bool
self._parsed_url = None # type: Optional[urllib_parse.ParseResult]
self._parsed_url = None # type: Optional[URI]
self._setup_cfg = None # type: Optional[STRING_TYPE]
self._setup_py = None # type: Optional[STRING_TYPE]
self._pyproject_toml = None # type: Optional[STRING_TYPE]
@@ -301,7 +297,7 @@ class Line(object):
def line_for_ireq(self):
# type: () -> STRING_TYPE
line = "" # type: STRING_TYPE
if self.is_file or self.is_url and not self.is_vcs:
if self.is_file or self.is_remote_url and not self.is_vcs:
scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri"
local_line = next(
iter(
@@ -340,7 +336,7 @@ class Line(object):
if self.editable:
if not line:
if self.is_path or self.is_file:
if not self.path:
if not self.path and self.url is not None:
line = pip_shims.shims.url_to_path(self.url)
else:
line = self.path
@@ -437,7 +433,7 @@ class Line(object):
# note: we need versions for direct dependencies at the very least
if (
self.is_file
or self.is_url
or self.is_remote_url
or self.is_path
or (self.is_vcs and not self.editable)
):
@@ -485,7 +481,7 @@ class Line(object):
self.parse_requirement()
if self._requirement is None and self._name is not None:
self._requirement = init_requirement(canonicalize_name(self.name))
if self.is_file or self.is_url and self._requirement is not None:
if self.is_file or self.is_remote_url and self._requirement is not None:
self._requirement.url = self.url
if (
self._requirement
@@ -549,8 +545,8 @@ class Line(object):
def parse_hashes(self):
# type: () -> "Line"
"""
Parse hashes from *self.line* and set them on the current object.
"""Parse hashes from *self.line* and set them on the current object.
:returns: Self
:rtype: `:class:~Line`
"""
@@ -567,17 +563,22 @@ class Line(object):
:rtype: :class:`~Line`
"""
extras = None
if "@" in self.line or self.is_vcs or self.is_url:
line = "{0}".format(self.line)
uri = URI.parse(line)
name = uri.name
if name:
self._name = name
if uri.host and uri.path and uri.scheme:
self.line = uri.to_string(
escape_password=False, direct=False, strip_ssh=uri.is_implicit_ssh
)
else:
line = "{0}".format(self.line)
if any([self.is_vcs, self.is_url, "@" in line]):
try:
if self.parsed_url.name:
self._name = self.parsed_url.name
if (
self.parsed_url.host
and self.parsed_url.path
and self.parsed_url.scheme
):
self.line = self.parsed_url.to_string(
escape_password=False,
direct=False,
strip_ssh=self.parsed_url.is_implicit_ssh,
)
except ValueError:
self.line, extras = pip_shims.shims._strip_extras(self.line)
else:
self.line, extras = pip_shims.shims._strip_extras(self.line)
@@ -595,37 +596,10 @@ class Line(object):
def get_url(self):
# type: () -> STRING_TYPE
"""Sets ``self.name`` if given a **PEP-508** style URL"""
line = self.line
try:
parsed = URI.parse(line)
line = parsed.to_string(escape_password=False, direct=False, strip_ref=True)
except ValueError:
pass
else:
self._parsed_url = parsed
return line
if self.vcs is not None and self.line.startswith("{0}+".format(self.vcs)):
_, _, _parseable = self.line.partition("+")
parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(_parseable))
line, _ = split_ref_from_uri(line)
else:
parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(line))
if "@" in self.line and parsed.scheme == "":
name, _, url = self.line.partition("@")
if self._name is None:
url = url.strip()
self._name = name.strip()
if is_valid_url(url):
self.is_direct_url = True
line = url.strip()
parsed = urllib_parse.urlparse(line)
url_path = parsed.path
if "@" in url_path:
url_path, _, _ = url_path.rpartition("@")
parsed = parsed._replace(path=url_path)
self._parsed_url = parsed
return line
"""Sets ``self.name`` if given a **PEP-508** style URL."""
return self.parsed_url.to_string(
escape_password=False, direct=False, strip_ref=True
)
@property
def name(self):
@@ -655,20 +629,16 @@ class Line(object):
@property
def url(self):
# type: () -> Optional[STRING_TYPE]
if self.uri is not None:
url = add_ssh_scheme_to_git_uri(self.uri)
else:
url = getattr(self.link, "url_without_fragment", None)
if url is not None:
url = add_ssh_scheme_to_git_uri(unquote(url))
if url is not None and self._parsed_url is None:
if self.vcs is not None:
_, _, _parseable = url.partition("+")
self._parsed_url = urllib_parse.urlparse(_parseable)
if self.is_vcs:
# strip the ref from the url
url, _ = split_ref_from_uri(url)
return url
try:
return self.parsed_url.to_string(
escape_password=False,
strip_ref=True,
strip_name=True,
strip_subdir=True,
strip_ssh=False,
)
except ValueError:
return None
@property
def link(self):
@@ -704,21 +674,36 @@ class Line(object):
# type: () -> bool
# Installable local files and installable non-vcs urls are handled
# as files, generally speaking
if is_vcs(self.line) or is_vcs(self.get_url()):
return True
try:
if is_vcs(self.line) or is_vcs(self.get_url()):
return True
except ValueError:
return False
return False
@property
def is_url(self):
# type: () -> bool
url = self.get_url()
try:
url = self.get_url()
except ValueError:
return False
if is_valid_url(url) or is_file_url(url):
return True
return False
@property
def is_remote_url(self):
# type: () -> bool
return self.is_url and self.parsed_url.host is not None
@property
def is_path(self):
# type: () -> bool
try:
line_url = self.get_url()
except ValueError:
line_url = None
if (
self.path
and (
@@ -730,7 +715,7 @@ class Line(object):
):
return True
elif (os.path.exists(self.line) and is_installable_file(self.line)) or (
os.path.exists(self.get_url()) and is_installable_file(self.get_url())
line_url and os.path.exists(line_url) and is_installable_file(line_url)
):
return True
return False
@@ -738,22 +723,32 @@ class Line(object):
@property
def is_file_url(self):
# type: () -> bool
url = self.get_url()
parsed_url_scheme = self._parsed_url.scheme if self._parsed_url else ""
if url and is_file_url(self.get_url()) or parsed_url_scheme == "file":
try:
url = self.get_url()
except ValueError:
return False
try:
parsed_url_scheme = self.parsed_url.scheme
except ValueError:
return False
if url and is_file_url(url) or parsed_url_scheme == "file":
return True
return False
@property
def is_file(self):
# type: () -> bool
try:
url = self.get_url()
except ValueError:
return False
if (
self.is_path
or (is_file_url(self.get_url()) and is_installable_file(self.get_url()))
or (is_file_url(url) and is_installable_file(url))
or (
self._parsed_url
and self._parsed_url.scheme == "file"
and is_installable_file(urllib_parse.urlunparse(self._parsed_url))
and self._parsed_url.is_file_url
and is_installable_file(self._parsed_url.url_without_fragment_or_ref)
)
):
return True
@@ -762,7 +757,13 @@ class Line(object):
@property
def is_named(self):
# type: () -> bool
return not (self.is_file_url or self.is_url or self.is_file or self.is_vcs)
return not (
self.is_file_url
or self.is_url
or self.is_file
or self.is_vcs
or self.is_direct_url
)
@property
def ref(self):
@@ -781,7 +782,11 @@ class Line(object):
@property
def is_installable(self):
# type: () -> bool
possible_paths = (self.line, self.get_url(), self.path, self.base_path)
try:
url = self.get_url()
except ValueError:
url = None
possible_paths = (self.line, url, self.path, self.base_path)
return any(is_installable_file(p) for p in possible_paths if p is not None)
@property
@@ -794,7 +799,7 @@ class Line(object):
# type: () -> SetupInfo
setup_info = None
with pip_shims.shims.global_tempdir_manager():
setup_info = SetupInfo.from_ireq(self.ireq)
setup_info = SetupInfo.from_ireq(self.ireq, subdir=self.subdirectory)
if not setup_info.name:
setup_info.get_info()
return setup_info
@@ -850,6 +855,21 @@ class Line(object):
self._vcsrepo = self._get_vcsrepo()
return self._vcsrepo
@property
def parsed_url(self):
# type: () -> URI
if self._parsed_url is None:
self._parsed_url = URI.parse(self.line)
return self._parsed_url
@property
def is_direct_url(self):
# type: () -> bool
try:
return self.is_url and self.parsed_url.is_direct_url
except ValueError:
return self.is_url and bool(DIRECT_URL_RE.match(self.line))
@cached_property
def metadata(self):
# type: () -> Dict[Any, Any]
@@ -886,8 +906,8 @@ class Line(object):
ireq = self.ireq
wheel_kwargs = self.wheel_kwargs.copy()
wheel_kwargs["src_dir"] = repo.checkout_directory
ireq.ensure_has_source_dir(wheel_kwargs["src_dir"])
with pip_shims.shims.global_tempdir_manager(), temp_path():
ireq.ensure_has_source_dir(wheel_kwargs["src_dir"])
sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)]
setupinfo = SetupInfo.create(
repo.checkout_directory,
@@ -907,7 +927,7 @@ class Line(object):
ireq = pip_shims.shims.install_req_from_line(line)
if self.is_named:
ireq = pip_shims.shims.install_req_from_line(self.line)
if self.is_file or self.is_url:
if self.is_file or self.is_remote_url:
ireq.link = self.link
if self.extras and not ireq.extras:
ireq.extras = set(self.extras)
@@ -1001,10 +1021,11 @@ class Line(object):
# type: () -> "Line"
if self._name is None:
name = None
if self.link is not None:
if self.link is not None and self.line_is_installable:
name = self._parse_name_from_link()
if name is None and (
(self.is_url or self.is_artifact or self.is_vcs) and self._parsed_url
(self.is_remote_url or self.is_artifact or self.is_vcs)
and self._parsed_url
):
if self._parsed_url.fragment:
_, _, name = self._parsed_url.fragment.partition("egg=")
@@ -1013,7 +1034,7 @@ class Line(object):
name, _, _ = name.partition("&")
if name is None and self.is_named:
name = self._parse_name_from_line()
elif name is None and self.is_file or self.is_url or self.is_path:
elif name is None and self.is_file or self.is_remote_url or self.is_path:
if self.is_local:
name = self._parse_name_from_path()
if name is not None:
@@ -1052,10 +1073,10 @@ class Line(object):
# else:
# req.link = self.link
if self.ref and self._requirement is not None:
self._requirement.revision = self.ref
if self._vcsrepo is not None:
self._requirement.revision = self._vcsrepo.get_commit_hash()
else:
self._requirement.revision = self.ref
with pip_shims.shims.global_tempdir_manager():
self._requirement.revision = self._vcsrepo.get_commit_hash()
return self._requirement
def parse_requirement(self):
@@ -1107,53 +1128,64 @@ class Line(object):
def parse_link(self):
# type: () -> "Line"
parsed_url = None # type: Optional[URI]
if not is_valid_url(self.line) and (
self.line.startswith("./")
or (os.path.exists(self.line) or os.path.isabs(self.line))
if (
not is_valid_url(self.line)
and is_installable_file(os.path.abspath(self.line))
and (
self.line.startswith("./")
or (os.path.exists(self.line) or os.path.isabs(self.line))
)
):
url = pip_shims.shims.path_to_url(os.path.abspath(self.line))
parsed_url = URI.parse(url)
elif is_valid_url(self.line) or is_vcs(self.line) or is_file_url(self.line):
parsed_url = URI.parse(self.line)
if parsed_url is not None:
line = parsed_url.to_string(
escape_password=False, direct=False, strip_ref=True, strip_ssh=False
)
if parsed_url.is_vcs:
self.vcs, _ = parsed_url.scheme.split("+")
if parsed_url.is_file_url:
self.is_local = True
parsed_link = parsed_url.as_link
self._ref = parsed_url.ref
self.uri = parsed_url.bare_url
if parsed_url.name:
self._name = parsed_url.name
if parsed_url.extras:
self.extras = tuple(sorted(set(parsed_url.extras)))
self._parsed_url = parsed_url = URI.parse(url)
elif any(
[
is_valid_url(self.line),
is_vcs(self.line),
is_file_url(self.line),
self.is_direct_url,
]
):
parsed_url = self.parsed_url
if parsed_url is None or (
parsed_url.is_file_url and not parsed_url.is_installable
):
return None
if parsed_url.is_vcs:
self.vcs, _ = parsed_url.scheme.split("+")
if parsed_url.is_file_url:
self.is_local = True
parsed_link = parsed_url.as_link
self._ref = parsed_url.ref
self.uri = parsed_url.bare_url
if parsed_url.name:
self._name = parsed_url.name
if parsed_url.extras:
self.extras = tuple(sorted(set(parsed_url.extras)))
self._link = parsed_link
vcs, prefer, relpath, path, uri, link = FileRequirement.get_link_from_line(
self.line
)
ref = None
if link is not None and "@" in unquote(link.path) and uri is not None:
uri, _, ref = unquote(uri).rpartition("@")
if relpath is not None and "@" in relpath:
relpath, _, ref = relpath.rpartition("@")
if path is not None and "@" in path:
path, _ = split_ref_from_uri(path)
link_url = link.url_without_fragment
if "@" in link_url:
link_url, _ = split_ref_from_uri(link_url)
self.preferred_scheme = prefer
self.relpath = relpath
self.path = path
# self.uri = uri
if prefer in ("path", "relpath") or uri.startswith("file"):
self.is_local = True
if parsed_url.is_vcs or parsed_url.is_direct_url and parsed_link:
self._link = parsed_link
vcs, prefer, relpath, path, uri, link = FileRequirement.get_link_from_line(
self.line
)
ref = None
if link is not None and "@" in unquote(link.path) and uri is not None:
uri, _, ref = unquote(uri).rpartition("@")
if relpath is not None and "@" in relpath:
relpath, _, ref = relpath.rpartition("@")
if path is not None and "@" in path:
path, _ = split_ref_from_uri(path)
link_url = link.url_without_fragment
if "@" in link_url:
link_url, _ = split_ref_from_uri(link_url)
self.preferred_scheme = prefer
self.relpath = relpath
self.path = path
# self.uri = uri
if prefer in ("path", "relpath") or uri.startswith("file"):
self.is_local = True
if parsed_url.is_vcs or parsed_url.is_direct_url and parsed_link:
self._link = parsed_link
else:
self._link = link
else:
self._link = link
return self
def parse_markers(self):
@@ -1206,26 +1238,50 @@ class Line(object):
@property
def line_is_installable(self):
# type: () -> bool
"""
This is a safeguard against decoy requirements when a user installs a package
whose name coincides with the name of a folder in the cwd, e.g. install *alembic*
when there is a folder called *alembic* in the working directory.
"""This is a safeguard against decoy requirements when a user installs
a package whose name coincides with the name of a folder in the cwd,
e.g. install *alembic* when there is a folder called *alembic* in the
working directory.
In this case we first need to check that the given requirement is a valid
URL, VCS requirement, or installable filesystem path before deciding to treat it
as a file requirement over a named requirement.
In this case we first need to check that the given requirement
is a valid URL, VCS requirement, or installable filesystem path
before deciding to treat it as a file requirement over a named
requirement.
"""
line = self.line
direct_url_match = DIRECT_URL_RE.match(line)
if direct_url_match:
match_dict = direct_url_match.groupdict()
auth = ""
username = match_dict.get("username", None)
password = match_dict.get("password", None)
port = match_dict.get("port", None)
path = match_dict.get("path", None)
ref = match_dict.get("ref", None)
if username is not None:
auth = "{0}".format(username)
if password:
auth = "{0}:{1}".format(auth, password) if auth else password
line = match_dict.get("host", "")
if auth:
line = "{auth}@{line}".format(auth=auth, line=line)
if port:
line = "{line}:{port}".format(line=line, port=port)
if path:
line = "{line}{pathsep}{path}".format(
line=line, pathsep=match_dict["pathsep"], path=path
)
if ref:
line = "{line}@{ref}".format(line=line, ref=ref)
line = "{scheme}{line}".format(scheme=match_dict["scheme"], line=line)
if is_file_url(line):
link = create_link(line)
line = link.url_without_fragment
line, _ = split_ref_from_uri(line)
if (
is_vcs(line)
or (
is_valid_url(line)
and (not is_file_url(line) or is_installable_file(line))
)
or (not is_file_url(line) and is_valid_url(line))
or (is_file_url(line) and is_installable_file(line))
or is_installable_file(line)
):
return True
@@ -1253,6 +1309,8 @@ class Line(object):
raise RequirementError(
"Supplied requirement is not installable: {0!r}".format(self.line)
)
elif self.is_named and self._name is None:
self.parse_name()
self.parse_link()
# self.parse_requirement()
# self.parse_ireq()
@@ -1385,6 +1443,7 @@ class FileRequirement(object):
pyproject_backend = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE]
#: PyProject Path
pyproject_path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE]
subdirectory = attr.ib(default=None) # type: Optional[STRING_TYPE]
#: Setup metadata e.g. dependencies
_setup_info = attr.ib(default=None, cmp=True) # type: Optional[SetupInfo]
_has_hashed_name = attr.ib(default=False, cmp=True) # type: bool
@@ -1551,8 +1610,6 @@ class FileRequirement(object):
@property
def setup_info(self):
# type: () -> Optional[SetupInfo]
from .setup_info import SetupInfo
if self._setup_info is None and self.parsed_line:
if self.parsed_line and self._parsed_line and self.parsed_line.setup_info:
if (
@@ -1566,7 +1623,9 @@ class FileRequirement(object):
self.parsed_line.ireq and not self.parsed_line.is_wheel
):
with pip_shims.shims.global_tempdir_manager():
self._setup_info = SetupInfo.from_ireq(self.parsed_line.ireq)
self._setup_info = SetupInfo.from_ireq(
self.parsed_line.ireq, subdir=self.subdirectory
)
else:
if self.link and not self.link.is_wheel:
self._setup_info = Line(self.line_part).setup_info
@@ -1889,7 +1948,6 @@ class VCSRequirement(FileRequirement):
#: vcs reference name (branch / commit / tag)
ref = attr.ib(default=None) # type: Optional[STRING_TYPE]
#: Subdirectory to use for installation if applicable
subdirectory = attr.ib(default=None) # type: Optional[STRING_TYPE]
_repo = attr.ib(default=None) # type: Optional[VCSRepository]
_base_line = attr.ib(default=None) # type: Optional[STRING_TYPE]
name = attr.ib() # type: STRING_TYPE
@@ -1960,20 +2018,18 @@ class VCSRequirement(FileRequirement):
with pip_shims.shims.global_tempdir_manager():
self._parsed_line._setup_info.get_info()
return self._parsed_line.setup_info
subdir = self.subdirectory or self.parsed_line.subdirectory
if self._repo:
from .setup_info import SetupInfo
with pip_shims.shims.global_tempdir_manager():
self._setup_info = SetupInfo.from_ireq(
Line(self._repo.checkout_directory).ireq
Line(self._repo.checkout_directory).ireq, subdir=subdir
)
self._setup_info.get_info()
return self._setup_info
ireq = self.parsed_line.ireq
from .setup_info import SetupInfo
with pip_shims.shims.global_tempdir_manager():
self._setup_info = SetupInfo.from_ireq(ireq)
self._setup_info = SetupInfo.from_ireq(ireq, subdir=subdir)
return self._setup_info
@setup_info.setter
@@ -2003,7 +2059,7 @@ class VCSRequirement(FileRequirement):
)
req = init_requirement(canonicalize_name(self.name))
req.editable = self.editable
if not getattr(req, "url"):
if not getattr(req, "url", None):
if url is not None:
url = add_ssh_scheme_to_git_uri(url)
elif self.uri is not None:
@@ -2114,21 +2170,18 @@ class VCSRequirement(FileRequirement):
def get_commit_hash(self):
# type: () -> STRING_TYPE
hash_ = None
hash_ = self.repo.get_commit_hash()
with pip_shims.shims.global_tempdir_manager():
hash_ = self.repo.get_commit_hash()
return hash_
def update_repo(self, src_dir=None, ref=None):
# type: (Optional[STRING_TYPE], Optional[STRING_TYPE]) -> STRING_TYPE
if ref:
self.ref = ref
else:
if self.ref:
ref = self.ref
repo_hash = None
if not self.is_local and ref is not None:
self.repo.checkout_ref(ref)
repo_hash = self.repo.get_commit_hash()
if not self.is_local and self.ref is not None:
self.repo.checkout_ref(self.ref)
repo_hash = self.get_commit_hash()
if self.req:
self.req.revision = repo_hash
return repo_hash
@@ -2144,7 +2197,8 @@ class VCSRequirement(FileRequirement):
self.req = self.parsed_line.requirement
else:
self.req = self.get_requirement()
revision = self.req.revision = vcsrepo.get_commit_hash()
with pip_shims.shims.global_tempdir_manager():
revision = self.req.revision = vcsrepo.get_commit_hash()
# Remove potential ref in the end of uri after ref is parsed
if self.link and "@" in self.link.show_url and self.uri and "@" in self.uri:
@@ -2237,7 +2291,7 @@ class VCSRequirement(FileRequirement):
@property
def line_part(self):
# type: () -> STRING_TYPE
"""requirements.txt compatible line part sans-extras"""
"""requirements.txt compatible line part sans-extras."""
base = "" # type: STRING_TYPE
if self.is_local:
base_link = self.link
@@ -2620,7 +2674,8 @@ class Requirement(object):
None
) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]]
if (
(parsed_line.is_file and parsed_line.is_installable) or parsed_line.is_url
(parsed_line.is_file and parsed_line.is_installable)
or parsed_line.is_remote_url
) and not parsed_line.is_vcs:
r = file_req_from_parsed_line(parsed_line)
elif parsed_line.is_vcs:
@@ -2956,7 +3011,6 @@ class Requirement(object):
elif self.line_instance and self.line_instance.setup_info is not None:
info_dict = self.line_instance.setup_info.as_dict()
else:
from .setup_info import SetupInfo
if not finder:
from .dependencies import get_finder
@@ -3095,3 +3149,8 @@ def named_req_from_parsed_line(parsed_line):
parsed_line=parsed_line,
)
return NamedRequirement.from_line(parsed_line.line)
if __name__ == "__main__":
line = Line("vistir@ git+https://github.com/sarugaku/vistir.git@master")
print(line)
+291 -40
View File
@@ -23,9 +23,10 @@ import six
from appdirs import user_cache_dir
from distlib.wheel import Wheel
from packaging.markers import Marker
from pip_shims.utils import call_function_with_correct_args
from six.moves import configparser
from six.moves.urllib.parse import unquote, urlparse, urlunparse
from vistir.compat import FileNotFoundError, Iterable, Mapping, Path, lru_cache
from vistir.compat import FileNotFoundError, Iterable, Mapping, Path, finalize, lru_cache
from vistir.contextmanagers import cd, temp_path
from vistir.misc import run
from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p, rmtree
@@ -695,6 +696,7 @@ class Analyzer(ast.NodeVisitor):
self.assignments = {}
self.binOps = []
self.binOps_map = {}
self.recurse = True
super(Analyzer, self).__init__()
def generic_visit(self, node):
@@ -709,6 +711,15 @@ class Analyzer(ast.NodeVisitor):
self.assignments.update(ast_unparse(node, initial_mapping=True))
super(Analyzer, self).generic_visit(node)
@contextlib.contextmanager
def no_recurse(self):
original_recurse_val = self.recurse
try:
self.recurse = False
yield
finally:
self.recurse = original_recurse_val
def visit_BinOp(self, node):
node = ast_unparse(node, initial_mapping=True)
self.binOps.append(node)
@@ -727,6 +738,202 @@ class Analyzer(ast.NodeVisitor):
iter(k for k in self.assignments if getattr(k, "id", "") == match.id), None
)
def generic_unparse(self, item):
if any(isinstance(item, k) for k in AST_BINOP_MAP.keys()):
return AST_BINOP_MAP[type(item)]
elif any(isinstance(item, k) for k in AST_COMPARATORS.keys()):
return AST_COMPARATORS[type(item)]
return item
def unparse(self, item):
unparser = getattr(
self, "unparse_{0}".format(item.__class__.__name__), self.generic_unparse
)
return unparser(item)
def unparse_Dict(self, item):
# unparsed = dict(zip(unparse(item.keys), unparse(item.values)))
return dict(
(self.unparse(k), self.unparse(v)) for k, v in zip(item.keys, item.values)
)
def unparse_List(self, item):
return [self.unparse(el) for el in item.elts]
def unparse_Tuple(self, item):
return tuple([self.unparse(el) for el in item.elts])
def unparse_Str(self, item):
return item.s
def unparse_Subscript(self, item):
unparsed = self.unparse(item.value)
if isinstance(item.slice, ast.Index):
try:
unparsed = unparsed[self.unparse(item.slice.value)]
except KeyError:
# not everything can be looked up before runtime
unparsed = item
return unparsed
def unparse_Num(self, item):
return item.n
def unparse_BinOp(self, item):
if item in self.binOps_map:
unparsed = self.binOps_map[item]
else:
right_item = self.unparse(item.right)
left_item = self.unparse(item.left)
op = getattr(item, "op", None)
op_func = self.unparse(op) if op is not None else op
try:
unparsed = op_func(left_item, right_item)
except Exception:
unparsed = (left_item, op_func, right_item)
return unparsed
def unparse_Name(self, item):
unparsed = item.id
if not self.recurse:
return unparsed
if item in self.assignments and self.recurse:
items = self.unparse(self.assignments[item])
unparsed = items.get(item.id, item.id)
else:
assignment = self.match_assignment_name(item)
if assignment is not None:
items = self.unparse(self.assignments[assignment])
unparsed = items.get(item.id, item.id)
return unparsed
def unparse_NameConstant(self, item):
return item.value
def unparse_Constant(self, item):
return item.value
def unparse_Ellipsis(self, item):
return item.value
def unparse_Attribute(self, item):
attr_name = getattr(item, "value", None)
attr_attr = getattr(item, "attr", None)
name = None
name = self.unparse(attr_name) if attr_name is not None else attr_attr
if attr_name and not self.recurse:
name = attr_name
elif name and attr_attr:
if isinstance(name, six.string_types):
unparsed = ".".join([item for item in (name, attr_attr) if item])
else:
unparsed = item
elif attr_attr and not name:
unparsed = attr_attr
else:
unparsed = name if not unparsed else unparsed
return unparsed
def unparse_Compare(self, item):
if isinstance(item.left, ast.Attribute) or isinstance(item.left, ast.Str):
import importlib
left = unparse(item.left)
if "." in left:
name, _, val = left.rpartition(".")
left = getattr(importlib.import_module(name), val, left)
comparators = []
for comparator in item.comparators:
right = self.unparse(comparator)
if isinstance(comparator, ast.Attribute) and "." in right:
name, _, val = right.rpartition(".")
right = getattr(importlib.import_module(name), val, right)
comparators.append(right)
unparsed = (left, self.unparse(item.ops), comparators)
else:
unparsed = item
return unparsed
def unparse_IfExp(self, item):
ops, truth_vals = [], []
if isinstance(item.test, ast.Compare):
left, ops, right = self.unparse(item.test)
else:
result = self.unparse(item.test)
if isinstance(result, dict):
k, v = result.popitem()
if not v:
truth_vals = [False]
for i, op in enumerate(ops):
if i == 0:
truth_vals.append(op(left, right[i]))
else:
truth_vals.append(op(right[i - 1], right[i]))
if all(truth_vals):
unparsed = self.unparse(item.body)
else:
unparsed = self.unparse(item.orelse)
return unparsed
def unparse_Call(self, item):
unparsed = {}
if isinstance(item.func, (ast.Name, ast.Attribute)):
func_name = self.unparse(item.func)
else:
try:
func_name = self.unparse(item.func)
except Exception:
func_name = None
if not func_name:
return {}
if isinstance(func_name, dict):
unparsed.update(func_name)
func_name = next(iter(func_name.keys()))
else:
unparsed[func_name] = {}
for key in ("kwargs", "keywords"):
val = getattr(item, key, [])
if val is None:
continue
for keyword in self.unparse(val):
unparsed[func_name].update(self.unparse(keyword))
return unparsed
def unparse_keyword(self, item):
return {self.unparse(item.arg): self.unparse(item.value)}
def unparse_Assign(self, item):
# XXX: DO NOT UNPARSE THIS
# XXX: If we unparse this it becomes impossible to map it back
# XXX: To the original node in the AST so we can find the
# XXX: Original reference
with self.no_recurse():
target = self.unparse(next(iter(item.targets)))
val = self.unparse(item.value)
if isinstance(target, (tuple, set, list)):
unparsed = dict(zip(target, val))
else:
unparsed = {target: val}
return unparsed
def unparse_Mapping(self, item):
unparsed = {}
for k, v in item.items():
try:
unparsed[self.unparse(k)] = self.unparse(v)
except TypeError:
unparsed[k] = self.unparse(v)
return unparsed
def unparse_list(self, item):
return type(item)([self.unparse(el) for el in item])
def unparse_tuple(self, item):
return self.unparse_list(item)
def unparse_str(self, item):
return item
def parse_function_names(self, should_retry=True, function_map=None):
if function_map is None:
function_map = {}
@@ -759,6 +966,17 @@ class Analyzer(ast.NodeVisitor):
)
return self.resolved_function_names
def parse_setup_function(self):
setup = {} # type: Dict[Any, Any]
self.unmap_binops()
function_names = self.parse_functions()
if "setup" in function_names:
setup = self.unparse(function_names["setup"])
keys = list(setup.keys())
if len(keys) == 1 and keys[0] is None:
_, setup = setup.popitem()
return setup
def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # noqa:C901
# type: (Any, bool, Optional[Analyzer], bool) -> Union[List[Any], Dict[Any, Any], Tuple[Any, ...], STRING_TYPE]
@@ -895,15 +1113,21 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no
func_name = unparse(item.func)
except Exception:
func_name = None
if func_name and not isinstance(func_name, dict):
unparsed[func_name] = {}
if isinstance(func_name, dict):
unparsed.update(func_name)
func_name = next(iter(func_name.keys()))
for keyword in getattr(item, "keywords", []):
unparsed[func_name].update(unparse(keyword))
elif func_name:
unparsed[func_name] = {}
for keyword in getattr(item, "keywords", []):
unparsed[func_name].update(unparse(keyword))
if func_name:
for key in ("kwargs", "keywords"):
val = getattr(item, key, [])
if val is None:
continue
if isinstance(val, ast.Name):
unparsed[func_name] = val
else:
for keyword in unparse(val):
unparsed[func_name].update(unparse(keyword))
elif isinstance(item, ast.keyword):
unparsed = {unparse(item.arg): unparse(item.value)}
elif isinstance(item, ast.Assign):
@@ -978,6 +1202,9 @@ def ast_parse_setup_py(path):
function_names = ast_analyzer.parse_functions()
if "setup" in function_names:
setup = ast_unparse(function_names["setup"], analyzer=ast_analyzer)
keys = list(setup.keys())
if len(keys) == 1 and keys[0] is None:
_, setup = setup.popitem()
return setup
@@ -1111,29 +1338,34 @@ class Extra(object):
return {self.name: tuple([r.requirement for r in self.requirements])}
@attr.s(slots=True, cmp=True, hash=True)
@attr.s(slots=True, eq=True, hash=True)
class SetupInfo(object):
name = attr.ib(default=None, cmp=True) # type: STRING_TYPE
base_dir = attr.ib(default=None, cmp=True, hash=False) # type: STRING_TYPE
_version = attr.ib(default=None, cmp=True) # type: STRING_TYPE
name = attr.ib(default=None, eq=True) # type: STRING_TYPE
base_dir = attr.ib(default=None, eq=True, hash=False) # type: STRING_TYPE
_version = attr.ib(default=None, eq=True) # type: STRING_TYPE
_requirements = attr.ib(
type=frozenset, factory=frozenset, cmp=True, hash=True
type=frozenset, factory=frozenset, eq=True, hash=True
) # type: Optional[frozenset]
build_requires = attr.ib(default=None, cmp=True) # type: Optional[Tuple]
build_backend = attr.ib(cmp=True) # type: STRING_TYPE
setup_requires = attr.ib(default=None, cmp=True) # type: Optional[Tuple]
build_requires = attr.ib(default=None, eq=True) # type: Optional[Tuple]
build_backend = attr.ib(eq=True) # type: STRING_TYPE
setup_requires = attr.ib(default=None, eq=True) # type: Optional[Tuple]
python_requires = attr.ib(
default=None, cmp=True
default=None, eq=True
) # type: Optional[packaging.specifiers.SpecifierSet]
_extras_requirements = attr.ib(default=None, cmp=True) # type: Optional[Tuple]
setup_cfg = attr.ib(type=Path, default=None, cmp=True, hash=False)
setup_py = attr.ib(type=Path, default=None, cmp=True, hash=False)
pyproject = attr.ib(type=Path, default=None, cmp=True, hash=False)
_extras_requirements = attr.ib(default=None, eq=True) # type: Optional[Tuple]
setup_cfg = attr.ib(type=Path, default=None, eq=True, hash=False)
setup_py = attr.ib(type=Path, default=None, eq=True, hash=False)
pyproject = attr.ib(type=Path, default=None, eq=True, hash=False)
ireq = attr.ib(
default=None, cmp=True, hash=False
default=None, eq=True, hash=False
) # type: Optional[InstallRequirement]
extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict, cmp=False, hash=False)
extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict, eq=False, hash=False)
metadata = attr.ib(default=None) # type: Optional[Tuple[STRING_TYPE]]
stack = attr.ib(default=None, eq=False) # type: Optional[ExitStack]
_finalizer = attr.ib(default=None, eq=False) # type: Any
def __attrs_post_init__(self):
self._finalizer = finalize(self, self.stack.close)
@build_backend.default
def get_build_backend(self):
@@ -1399,8 +1631,8 @@ build-backend = "{1}"
# type: () -> Dict[S, Any]
"""Wipe existing distribution info metadata for rebuilding.
Erases metadata from **self.egg_base** and unsets **self.requirements**
and **self.extras**.
Erases metadata from **self.egg_base** and unsets
**self.requirements** and **self.extras**.
"""
for metadata_dir in os.listdir(self.egg_base):
shutil.rmtree(metadata_dir, ignore_errors=True)
@@ -1422,7 +1654,8 @@ build-backend = "{1}"
def get_egg_metadata(self, metadata_dir=None, metadata_type=None):
# type: (Optional[AnyStr], Optional[AnyStr]) -> Dict[Any, Any]
"""Given a metadata directory, return the corresponding metadata dictionary.
"""Given a metadata directory, return the corresponding metadata
dictionary.
:param Optional[str] metadata_dir: Root metadata path, default: `os.getcwd()`
:param Optional[str] metadata_type: Type of metadata to search for, default None
@@ -1586,10 +1819,13 @@ build-backend = "{1}"
return None
if ireq.link.is_wheel:
return None
if not finder:
from .dependencies import get_finder
session, finder = get_finder()
stack = ExitStack()
if not session:
cmd = pip_shims.shims.InstallCommand()
options, _ = cmd.parser.parse_args([])
session = cmd._build_session(options)
finder = cmd._build_package_finder(options, session)
tempdir_manager = stack.enter_context(pip_shims.shims.global_tempdir_manager())
vcs, uri = split_vcs_method_from_uri(unquote(ireq.link.url_without_fragment))
parsed = urlparse(uri)
if "file" in parsed.scheme:
@@ -1599,7 +1835,9 @@ build-backend = "{1}"
parsed = parsed._replace(path=url_path)
uri = urlunparse(parsed)
path = None
is_file = False
if ireq.link.scheme == "file" or uri.startswith("file://"):
is_file = True
if "file:/" in uri and "file:///" not in uri:
uri = uri.replace("file:/", "file:///")
path = pip_shims.shims.url_to_path(uri)
@@ -1608,7 +1846,11 @@ build-backend = "{1}"
ireq.link, "is_vcs", getattr(ireq.link, "is_artifact", False)
)
is_vcs = True if vcs else is_artifact_or_vcs
if not (ireq.editable and pip_shims.shims.is_file_url(ireq.link) and is_vcs):
if is_file and not is_vcs and path is not None and os.path.isdir(path):
target = os.path.join(kwargs["src_dir"], os.path.basename(path))
shutil.copytree(path, target)
ireq.source_dir = target
if not (ireq.editable and is_file and is_vcs):
if ireq.is_wheel:
only_download = True
download_dir = kwargs["wheel_download_dir"]
@@ -1624,27 +1866,33 @@ build-backend = "{1}"
build_location_func = getattr(ireq, "build_location", None)
if build_location_func is None:
build_location_func = getattr(ireq, "ensure_build_location", None)
build_location_func(kwargs["build_dir"])
ireq.ensure_has_source_dir(kwargs["src_dir"])
src_dir = ireq.source_dir
with pip_shims.shims.global_tempdir_manager():
if not ireq.source_dir:
build_kwargs = {"build_dir": kwargs["build_dir"], "autodelete": False}
call_function_with_correct_args(build_location_func, **build_kwargs)
ireq.ensure_has_source_dir(kwargs["src_dir"])
src_dir = ireq.source_dir
pip_shims.shims.shim_unpack(
link=ireq.link,
location=kwargs["src_dir"],
download_dir=download_dir,
ireq=ireq,
only_download=only_download,
session=session,
hashes=ireq.hashes(False),
progress_bar="off",
)
created = cls.create(
kwargs["src_dir"], subdirectory=subdir, ireq=ireq, kwargs=kwargs
ireq.source_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs, stack=stack
)
return created
@classmethod
def create(cls, base_dir, subdirectory=None, ireq=None, kwargs=None):
# type: (AnyStr, Optional[AnyStr], Optional[InstallRequirement], Optional[Dict[AnyStr, AnyStr]]) -> Optional[SetupInfo]
def create(
cls,
base_dir, # type: str
subdirectory=None, # type: Optional[str]
ireq=None, # type: Optional[InstallRequirement]
kwargs=None, # type: Optional[Dict[str, str]]
stack=None, # type: Optional[ExitStack]
):
# type: (...) -> Optional[SetupInfo]
if not base_dir or base_dir is None:
return None
@@ -1661,6 +1909,9 @@ build-backend = "{1}"
creation_kwargs["pyproject"] = pyproject
creation_kwargs["setup_py"] = setup_py
creation_kwargs["setup_cfg"] = setup_cfg
if stack is None:
stack = ExitStack()
creation_kwargs["stack"] = stack
if ireq:
creation_kwargs["ireq"] = ireq
created = cls(**creation_kwargs)
+26 -12
View File
@@ -10,6 +10,7 @@ from urllib3.util import parse_url as urllib3_parse
from urllib3.util.url import Url
from ..environment import MYPY_RUNNING
from ..utils import is_installable_file
from .utils import extras_to_string, parse_extras
if MYPY_RUNNING:
@@ -24,8 +25,7 @@ if MYPY_RUNNING:
def _get_parsed_url(url):
# type: (S) -> Url
"""
This is a stand-in function for `urllib3.util.parse_url`
"""This is a stand-in function for `urllib3.util.parse_url`
The orignal function doesn't handle special characters very well, this simply splits
out the authentication section, creates the parsed url, then puts the authentication
@@ -49,8 +49,7 @@ def _get_parsed_url(url):
def remove_password_from_url(url):
# type: (S) -> S
"""
Given a url, remove the password and insert 4 dashes
"""Given a url, remove the password and insert 4 dashes.
:param url: The url to replace the authentication in
:type url: S
@@ -108,12 +107,18 @@ class URI(object):
query_dict = omdict()
queries = query.split("&")
query_items = []
subdirectory = self.subdirectory if self.subdirectory else None
for q in queries:
key, _, val = q.partition("=")
val = unquote_plus(val.replace("+", " "))
query_items.append((key, val))
if key == "subdirectory" and not subdirectory:
subdirectory = val
else:
query_items.append((key, val))
query_dict.load(query_items)
return attr.evolve(self, query_dict=query_dict, query=query)
return attr.evolve(
self, query_dict=query_dict, subdirectory=subdirectory, query=query
)
def _parse_fragment(self):
# type: () -> URI
@@ -187,7 +192,10 @@ class URI(object):
subdir = None
if "&subdirectory" in url_part:
url_part, _, subdir = url_part.rpartition("&")
subdir = "&{0}".format(subdir.strip())
if "#egg=" not in url_part:
subdir = "#{0}".format(subdir.strip())
else:
subdir = "&{0}".format(subdir.strip())
return url_part.strip(), subdir
@classmethod
@@ -255,8 +263,8 @@ class URI(object):
strip_subdir=False, # type: bool
):
# type: (...) -> str
"""
Converts the current URI to a string, unquoting or escaping the password as needed
"""Converts the current URI to a string, unquoting or escaping the
password as needed.
:param escape_password: Whether to replace password with ``----``, default True
:param escape_password: bool, optional
@@ -295,9 +303,11 @@ class URI(object):
query = ""
if self.query:
query = "{query}?{self.query}".format(query=query, self=self)
subdir_prefix = "#"
if not direct:
if self.name and not strip_name:
fragment = "#egg={self.name_with_extras}".format(self=self)
subdir_prefix = "&"
elif not strip_name and (
self.extras and self.scheme and self.scheme.startswith("file")
):
@@ -308,8 +318,8 @@ class URI(object):
fragment = ""
query = "{query}{fragment}".format(query=query, fragment=fragment)
if self.subdirectory and not strip_subdir:
query = "{query}&subdirectory={self.subdirectory}".format(
query=query, self=self
query = "{query}{subdir_prefix}subdirectory={self.subdirectory}".format(
query=query, subdir_prefix=subdir_prefix, self=self
)
host_port_path = self.get_host_port_path(strip_ref=strip_ref)
url = "{self.scheme}://{auth}{host_port_path}{query}".format(
@@ -441,6 +451,11 @@ class URI(object):
# type: () -> str
return self.to_string(escape_password=False, unquote=False)
@property
def is_installable(self):
# type: () -> bool
return self.is_file_url and is_installable_file(self.bare_url)
@property
def is_vcs(self):
# type: () -> bool
@@ -477,7 +492,6 @@ def update_url_name_and_fragment(name_with_extras, ref, parsed_dict):
if fragment_extras:
parsed_extras = parsed_extras + tuple(parse_extras(fragment_extras))
name_with_extras = "{0}{1}".format(name, extras_to_string(parsed_extras))
parsed_dict["fragment"] = "egg={0}".format(name_with_extras)
elif (
parsed_dict.get("path") is not None and "&subdirectory" in parsed_dict["path"]
):
+2 -1
View File
@@ -106,7 +106,8 @@ class VCSRepository(object):
def get_commit_hash(self, ref=None):
# type: (Optional[str]) -> str
return self.repo_backend.get_revision(self.checkout_directory)
with pip_shims.shims.global_tempdir_manager():
return self.repo_backend.get_revision(self.checkout_directory)
@classmethod
def monkeypatch_pip(cls):
+1 -1
View File
@@ -121,7 +121,7 @@ def strip_ssh_from_git_uri(uri):
def add_ssh_scheme_to_git_uri(uri):
# type: (S) -> S
"""Cleans VCS uris from pipenv.patched.notpip format"""
"""Cleans VCS uris from pip format"""
if isinstance(uri, six.string_types):
# Add scheme for parsing purposes, this is also what pip does
if uri.startswith("git+") and "://" not in uri:
@@ -1,11 +1,12 @@
The MIT License
Copyright 2013-2018 William Pearson
Copyright 2013-2019 William Pearson
Copyright 2015-2016 Julien Enselme
Copyright 2016 Google Inc.
Copyright 2017 Samuel Vasko
Copyright 2017 Nate Prewitt
Copyright 2017 Jack Evans
Copyright 2019 Filippo Broggini
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
+5 -1
View File
@@ -6,16 +6,20 @@ Released under the MIT license.
from toml import encoder
from toml import decoder
__version__ = "0.10.0"
__version__ = "0.10.1"
_spec_ = "0.5.0"
load = decoder.load
loads = decoder.loads
TomlDecoder = decoder.TomlDecoder
TomlDecodeError = decoder.TomlDecodeError
TomlPreserveCommentDecoder = decoder.TomlPreserveCommentDecoder
dump = encoder.dump
dumps = encoder.dumps
TomlEncoder = encoder.TomlEncoder
TomlArraySeparatorEncoder = encoder.TomlArraySeparatorEncoder
TomlPreserveInlineDictEncoder = encoder.TomlPreserveInlineDictEncoder
TomlNumpyEncoder = encoder.TomlNumpyEncoder
TomlPreserveCommentEncoder = encoder.TomlPreserveCommentEncoder
TomlPathlibEncoder = encoder.TomlPathlibEncoder
+6
View File
@@ -0,0 +1,6 @@
# content after the \
escapes = ['0', 'b', 'f', 'n', 'r', 't', '"']
# What it should be replaced by
escapedchars = ['\0', '\b', '\f', '\n', '\r', '\t', '\"']
# Used for substitution
escape_to_escapedchars = dict(zip(_escapes, _escapedchars))
+133 -26
View File
@@ -24,7 +24,7 @@ def _detect_pathlib_path(p):
def _ispath(p):
if isinstance(p, basestring):
if isinstance(p, (bytes, basestring)):
return True
return _detect_pathlib_path(p)
@@ -44,7 +44,7 @@ except NameError:
FNFError = IOError
TIME_RE = re.compile("([0-9]{2}):([0-9]{2}):([0-9]{2})(\.([0-9]{3,6}))?")
TIME_RE = re.compile(r"([0-9]{2}):([0-9]{2}):([0-9]{2})(\.([0-9]{3,6}))?")
class TomlDecodeError(ValueError):
@@ -66,6 +66,27 @@ class TomlDecodeError(ValueError):
_number_with_underscores = re.compile('([0-9])(_([0-9]))*')
class CommentValue(object):
def __init__(self, val, comment, beginline, _dict):
self.val = val
separator = "\n" if beginline else " "
self.comment = separator + comment
self._dict = _dict
def __getitem__(self, key):
return self.val[key]
def __setitem__(self, key, value):
self.val[key] = value
def dump(self, dump_value_func):
retstr = dump_value_func(self.val)
if isinstance(self.val, self._dict):
return self.comment + "\n" + unicode(retstr)
else:
return unicode(retstr) + self.comment
def _strictly_valid_num(n):
n = n.strip()
if not n:
@@ -96,6 +117,7 @@ def load(f, _dict=dict, decoder=None):
f: Path to the file to open, array of files to read into single dict
or a file descriptor
_dict: (optional) Specifies the class of the returned toml dictionary
decoder: The decoder to use
Returns:
Parsed toml file represented as a dictionary
@@ -120,9 +142,9 @@ def load(f, _dict=dict, decoder=None):
"existing file.")
raise FNFError(error_msg)
if decoder is None:
decoder = TomlDecoder()
decoder = TomlDecoder(_dict)
d = decoder.get_empty_table()
for l in f:
for l in f: # noqa: E741
if op.exists(l):
d.update(load(l, _dict, decoder))
else:
@@ -177,19 +199,30 @@ def loads(s, _dict=dict, decoder=None):
keygroup = False
dottedkey = False
keyname = 0
key = ''
prev_key = ''
line_no = 1
for i, item in enumerate(sl):
if item == '\r' and sl[i + 1] == '\n':
sl[i] = ' '
continue
if keyname:
key += item
if item == '\n':
raise TomlDecodeError("Key name found without value."
" Reached end of line.", original, i)
if openstring:
if item == openstrchar:
keyname = 2
openstring = False
openstrchar = ""
oddbackslash = False
k = 1
while i >= k and sl[i - k] == '\\':
oddbackslash = not oddbackslash
k += 1
if not oddbackslash:
keyname = 2
openstring = False
openstrchar = ""
continue
elif keyname == 1:
if item.isspace():
@@ -220,6 +253,8 @@ def loads(s, _dict=dict, decoder=None):
continue
if item == '=':
keyname = 0
prev_key = key[:-1].rstrip()
key = ''
dottedkey = False
else:
raise TomlDecodeError("Found invalid character in key name: '" +
@@ -272,12 +307,16 @@ def loads(s, _dict=dict, decoder=None):
if item == '#' and (not openstring and not keygroup and
not arrayoftables):
j = i
comment = ""
try:
while sl[j] != '\n':
comment += s[j]
sl[j] = ' '
j += 1
except IndexError:
break
if not openarr:
decoder.preserve_comment(line_no, prev_key, comment, beginline)
if item == '[' and (not openstring and not keygroup and
not arrayoftables):
if beginline:
@@ -308,12 +347,20 @@ def loads(s, _dict=dict, decoder=None):
sl[i] = ' '
else:
beginline = True
line_no += 1
elif beginline and sl[i] != ' ' and sl[i] != '\t':
beginline = False
if not keygroup and not arrayoftables:
if sl[i] == '=':
raise TomlDecodeError("Found empty keyname. ", original, i)
keyname = 1
key += item
if keyname:
raise TomlDecodeError("Key name found without value."
" Reached end of file.", original, len(s))
if openstring: # reached EOF and have an unterminated string
raise TomlDecodeError("Unterminated string found."
" Reached end of file.", original, len(s))
s = ''.join(sl)
s = s.split('\n')
multikey = None
@@ -323,6 +370,9 @@ def loads(s, _dict=dict, decoder=None):
for idx, line in enumerate(s):
if idx > 0:
pos += len(s[idx - 1]) + 1
decoder.embed_comments(idx, currentlevel)
if not multilinestr or multibackslash or '\n' not in multilinestr:
line = line.strip()
if line == "" and (not multikey or multibackslash):
@@ -333,9 +383,14 @@ def loads(s, _dict=dict, decoder=None):
else:
multilinestr += line
multibackslash = False
if len(line) > 2 and (line[-1] == multilinestr[0] and
line[-2] == multilinestr[0] and
line[-3] == multilinestr[0]):
closed = False
if multilinestr[0] == '[':
closed = line[-1] == ']'
elif len(line) > 2:
closed = (line[-1] == multilinestr[0] and
line[-2] == multilinestr[0] and
line[-3] == multilinestr[0])
if closed:
try:
value, vtype = decoder.load_value(multilinestr)
except ValueError as err:
@@ -663,7 +718,8 @@ class TomlDecoder(object):
while len(pair[-1]) and (pair[-1][0] != ' ' and pair[-1][0] != '\t' and
pair[-1][0] != "'" and pair[-1][0] != '"' and
pair[-1][0] != '[' and pair[-1][0] != '{' and
pair[-1] != 'true' and pair[-1] != 'false'):
pair[-1].strip() != 'true' and
pair[-1].strip() != 'false'):
try:
float(pair[-1])
break
@@ -671,6 +727,8 @@ class TomlDecoder(object):
pass
if _load_date(pair[-1]) is not None:
break
if TIME_RE.match(pair[-1]):
break
i += 1
prev_val = pair[-1]
pair = line.split('=', i)
@@ -704,16 +762,10 @@ class TomlDecoder(object):
pair[0] = levels[-1].strip()
elif (pair[0][0] == '"' or pair[0][0] == "'") and \
(pair[0][-1] == pair[0][0]):
pair[0] = pair[0][1:-1]
if len(pair[1]) > 2 and ((pair[1][0] == '"' or pair[1][0] == "'") and
pair[1][1] == pair[1][0] and
pair[1][2] == pair[1][0] and
not (len(pair[1]) > 5 and
pair[1][-1] == pair[1][0] and
pair[1][-2] == pair[1][0] and
pair[1][-3] == pair[1][0])):
k = len(pair[1]) - 1
while k > -1 and pair[1][k] == '\\':
pair[0] = _unescape(pair[0][1:-1])
k, koffset = self._load_line_multiline_str(pair[1])
if k > -1:
while k > -1 and pair[1][k + koffset] == '\\':
multibackslash = not multibackslash
k -= 1
if multibackslash:
@@ -734,6 +786,26 @@ class TomlDecoder(object):
else:
currentlevel[pair[0]] = value
def _load_line_multiline_str(self, p):
poffset = 0
if len(p) < 3:
return -1, poffset
if p[0] == '[' and (p.strip()[-1] != ']' and
self._load_array_isstrarray(p)):
newp = p[1:].strip().split(',')
while len(newp) > 1 and newp[-1][0] != '"' and newp[-1][0] != "'":
newp = newp[:-2] + [newp[-2] + ',' + newp[-1]]
newp = newp[-1]
poffset = len(p) - len(newp)
p = newp
if p[0] != '"' and p[0] != "'":
return -1, poffset
if p[1] != p[0] or p[2] != p[0]:
return -1, poffset
if len(p) > 5 and p[-1] == p[0] and p[-2] == p[0] and p[-3] == p[0]:
return -1, poffset
return len(p) - 1, poffset
def load_value(self, v, strictly_valid=True):
if not v:
raise ValueError("Empty value is invalid")
@@ -769,7 +841,8 @@ class TomlDecoder(object):
pass
if not oddbackslash:
if closed:
raise ValueError("Stuff after closed string. WTF?")
raise ValueError("Found tokens after a closed " +
"string. Invalid TOML.")
else:
if not triplequote or triplequotecount > 1:
closed = True
@@ -857,15 +930,18 @@ class TomlDecoder(object):
break
return not backslash
def _load_array_isstrarray(self, a):
a = a[1:-1].strip()
if a != '' and (a[0] == '"' or a[0] == "'"):
return True
return False
def load_array(self, a):
atype = None
retval = []
a = a.strip()
if '[' not in a[1:-1] or "" != a[1:-1].split('[')[0].strip():
strarray = False
tmpa = a[1:-1].strip()
if tmpa != '' and (tmpa[0] == '"' or tmpa[0] == "'"):
strarray = True
strarray = self._load_array_isstrarray(a)
if not a[1:-1].strip().startswith('{'):
a = a[1:-1].split(',')
else:
@@ -874,6 +950,7 @@ class TomlDecoder(object):
new_a = []
start_group_index = 1
end_group_index = 2
open_bracket_count = 1 if a[start_group_index] == '{' else 0
in_str = False
while end_group_index < len(a[1:]):
if a[end_group_index] == '"' or a[end_group_index] == "'":
@@ -884,9 +961,15 @@ class TomlDecoder(object):
in_str = not in_str
backslash_index -= 1
in_str = not in_str
if not in_str and a[end_group_index] == '{':
open_bracket_count += 1
if in_str or a[end_group_index] != '}':
end_group_index += 1
continue
elif a[end_group_index] == '}' and open_bracket_count > 1:
open_bracket_count -= 1
end_group_index += 1
continue
# Increase end_group_index by 1 to get the closing bracket
end_group_index += 1
@@ -943,3 +1026,27 @@ class TomlDecoder(object):
atype = ntype
retval.append(nval)
return retval
def preserve_comment(self, line_no, key, comment, beginline):
pass
def embed_comments(self, idx, currentlevel):
pass
class TomlPreserveCommentDecoder(TomlDecoder):
def __init__(self, _dict=dict):
self.saved_comments = {}
super(TomlPreserveCommentDecoder, self).__init__(_dict)
def preserve_comment(self, line_no, key, comment, beginline):
self.saved_comments[line_no] = (key, comment, beginline)
def embed_comments(self, idx, currentlevel):
if idx not in self.saved_comments:
return
key, comment, beginline = self.saved_comments[idx]
currentlevel[key] = CommentValue(currentlevel[key], comment, beginline,
self._dict)
+63 -9
View File
@@ -1,6 +1,7 @@
import datetime
import re
import sys
from decimal import Decimal
from toml.decoder import InlineTableDict
@@ -8,12 +9,13 @@ if sys.version_info >= (3,):
unicode = str
def dump(o, f):
def dump(o, f, encoder=None):
"""Writes out dict as toml to a file
Args:
o: Object to dump into toml
f: File descriptor where the toml should be stored
encoder: The ``TomlEncoder`` to use for constructing the output string
Returns:
String containing the toml corresponding to dictionary
@@ -24,7 +26,7 @@ def dump(o, f):
if not f.write:
raise TypeError("You can only dump an object to a file descriptor")
d = dumps(o)
d = dumps(o, encoder=encoder)
f.write(d)
return d
@@ -34,11 +36,22 @@ def dumps(o, encoder=None):
Args:
o: Object to dump into toml
preserve: Boolean parameter. If true, preserve inline tables.
encoder: The ``TomlEncoder`` to use for constructing the output string
Returns:
String containing the toml corresponding to dict
Examples:
```python
>>> import toml
>>> output = {
... 'a': "I'm a string",
... 'b': ["I'm", "a", "list"],
... 'c': 2400
... }
>>> toml.dumps(output)
'a = "I\'m a string"\nb = [ "I\'m", "a", "list",]\nc = 2400\n'
```
"""
retval = ""
@@ -46,7 +59,13 @@ def dumps(o, encoder=None):
encoder = TomlEncoder(o.__class__)
addtoretval, sections = encoder.dump_sections(o, "")
retval += addtoretval
outer_objs = [id(o)]
while sections:
section_ids = [id(section) for section in sections]
for outer_obj in outer_objs:
if outer_obj in section_ids:
raise ValueError("Circular reference detected")
outer_objs += section_ids
newsections = encoder.get_empty_table()
for section in sections:
addtoretval, addtosections = encoder.dump_sections(
@@ -96,7 +115,7 @@ def _dump_str(v):
def _dump_float(v):
return "{0:.16}".format(v).replace("e+0", "e+").replace("e-0", "e-")
return "{}".format(v).replace("e+0", "e+").replace("e-0", "e-")
def _dump_time(v):
@@ -119,6 +138,7 @@ class TomlEncoder(object):
bool: lambda v: unicode(v).lower(),
int: lambda v: v,
float: _dump_float,
Decimal: _dump_float,
datetime.datetime: lambda v: v.isoformat().replace('+00:00', 'Z'),
datetime.time: _dump_time,
datetime.date: lambda v: v.isoformat()
@@ -169,10 +189,7 @@ class TomlEncoder(object):
section = unicode(section)
qsection = section
if not re.match(r'^[A-Za-z0-9_-]+$', section):
if '"' in section:
qsection = "'" + section + "'"
else:
qsection = '"' + section + '"'
qsection = _dump_str(section)
if not isinstance(o[section], dict):
arrayoftables = False
if isinstance(o[section], list):
@@ -248,3 +265,40 @@ class TomlArraySeparatorEncoder(TomlEncoder):
t = s
retval += "]"
return retval
class TomlNumpyEncoder(TomlEncoder):
def __init__(self, _dict=dict, preserve=False):
import numpy as np
super(TomlNumpyEncoder, self).__init__(_dict, preserve)
self.dump_funcs[np.float16] = _dump_float
self.dump_funcs[np.float32] = _dump_float
self.dump_funcs[np.float64] = _dump_float
self.dump_funcs[np.int16] = self._dump_int
self.dump_funcs[np.int32] = self._dump_int
self.dump_funcs[np.int64] = self._dump_int
def _dump_int(self, v):
return "{}".format(int(v))
class TomlPreserveCommentEncoder(TomlEncoder):
def __init__(self, _dict=dict, preserve=False):
from toml.decoder import CommentValue
super(TomlPreserveCommentEncoder, self).__init__(_dict, preserve)
self.dump_funcs[CommentValue] = lambda v: v.dump(self.dump_value)
class TomlPathlibEncoder(TomlEncoder):
def _dump_pathlib_path(self, v):
return _dump_str(str(v))
def dump_value(self, v):
if (3, 4) <= sys.version_info:
import pathlib
if isinstance(v, pathlib.PurePath):
v = str(v)
return super(TomlPathlibEncoder, self).dump_value(v)
+10 -9
View File
@@ -1,7 +1,7 @@
appdirs==1.4.3
appdirs==1.4.4
backports.shutil_get_terminal_size==1.0.0
backports.weakref==1.0.post1
click==7.1.1
click==7.1.2
click-completion==0.5.2
click-didyoumean==0.0.3
colorama==0.4.3
@@ -11,7 +11,7 @@ delegator.py==0.1.1
python-dotenv==0.10.3
first==2.0.1
iso8601==0.1.12
jinja2==2.11.1
jinja2==2.11.2
markupsafe==1.1.1
parse==1.15.0
pathlib2==2.3.5
@@ -26,7 +26,7 @@ requests==2.23.0
idna==2.9
urllib3==1.25.9
certifi==2020.4.5.1
requirementslib==1.5.7
requirementslib==1.5.9
attrs==19.3.0
distlib==0.3.0
packaging==20.3
@@ -36,23 +36,24 @@ requirementslib==1.5.7
shellingham==1.3.2
six==1.14.0
semver==2.9.0
toml==0.10.0
toml==0.10.1
cached-property==1.5.1
vistir==0.5.0
vistir==0.5.1
pip-shims==0.5.2
contextlib2==0.6.0.post1
funcsigs==1.0.2
enum34==1.1.6
enum34==1.1.10
# yaspin==0.15.0
yaspin==0.14.3
cerberus==1.3.2
resolvelib==0.3.0
backports.functools_lru_cache==1.5
backports.functools_lru_cache==1.6.1
pep517==0.8.2
zipp==0.6.0
importlib_metadata==1.6.0
importlib-resources==1.4.0
importlib-resources==1.5.0
more-itertools==5.0.0
git+https://github.com/sarugaku/passa.git@master#egg=passa
orderedmultidict==1.0.1
dparse==0.5.0
python-dateutil==2.8.1
+1 -1
View File
@@ -36,7 +36,7 @@ from .misc import (
from .path import create_tracked_tempdir, create_tracked_tempfile, mkdir_p, rmtree
from .spin import create_spinner
__version__ = "0.5.0"
__version__ = "0.5.1"
__all__ = [
+15 -12
View File
@@ -60,7 +60,7 @@ from ctypes import (
py_object,
windll,
)
from ctypes.wintypes import LPCWSTR, LPWSTR
from ctypes.wintypes import HANDLE, LPCWSTR, LPWSTR
from itertools import count
import msvcrt
@@ -83,19 +83,18 @@ if IS_TYPE_CHECKING:
c_ssize_p = POINTER(c_ssize_t)
kernel32 = windll.kernel32
GetStdHandle = kernel32.GetStdHandle
ReadConsoleW = kernel32.ReadConsoleW
WriteConsoleW = kernel32.WriteConsoleW
GetLastError = kernel32.GetLastError
GetConsoleCursorInfo = kernel32.GetConsoleCursorInfo
SetConsoleCursorInfo = kernel32.SetConsoleCursorInfo
GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32))
CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))(
("CommandLineToArgvW", windll.shell32)
)
kernel32 = windll.kernel32
GetLastError = kernel32.GetLastError
GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32))
GetConsoleCursorInfo = kernel32.GetConsoleCursorInfo
GetStdHandle = kernel32.GetStdHandle
LocalFree = WINFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)(("LocalFree", windll.kernel32))
ReadConsoleW = kernel32.ReadConsoleW
SetConsoleCursorInfo = kernel32.SetConsoleCursorInfo
WriteConsoleW = kernel32.WriteConsoleW
# XXX: Added for cursor hiding on windows
STDOUT_HANDLE_ID = ctypes.c_ulong(-11)
@@ -354,7 +353,11 @@ if PY2:
def _get_windows_argv():
argc = c_int(0)
argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))
argv = [argv_unicode[i] for i in range(0, argc.value)]
try:
argv = [argv_unicode[i] for i in range(0, argc.value)]
finally:
LocalFree(argv_unicode)
del argv_unicode
if not hasattr(sys, "frozen"):
argv = argv[1:]
+1 -1
View File
@@ -35,7 +35,7 @@ if six.PY3:
_unichr = chr
bytes_chr = lambda code: bytes((code,))
else:
_unichr = unichr
_unichr = unichr # type: ignore
bytes_chr = chr
+85 -40
View File
@@ -29,11 +29,23 @@ __all__ = [
"TemporaryDirectory",
"NamedTemporaryFile",
"to_native_string",
"Iterable",
"samefile",
"Mapping",
"Sequence",
"Set",
"Hashable",
"MutableMapping",
"Container",
"Iterator",
"KeysView",
"ItemsView",
"MappingView",
"Iterable",
"Set",
"Sequence",
"Sized",
"ValuesView",
"MutableSet",
"MutableSequence",
"Callable",
"fs_encode",
"fs_decode",
"_fs_encode_errors",
@@ -45,23 +57,79 @@ if sys.version_info >= (3, 5): # pragma: no cover
else: # pragma: no cover
from pipenv.vendor.pathlib2 import Path
if six.PY3: # pragma: no cover
if sys.version_info >= (3, 4): # pragma: no cover
# Only Python 3.4+ is supported
from functools import lru_cache, partialmethod
from tempfile import NamedTemporaryFile
from shutil import get_terminal_size
from weakref import finalize
from collections.abc import (
Mapping,
Hashable,
MutableMapping,
Container,
Iterator,
KeysView,
ItemsView,
MappingView,
Iterable,
Set,
Sequence,
Sized,
ValuesView,
MutableSet,
MutableSequence,
Callable,
)
from os.path import samefile
else: # pragma: no cover
# Only Python 2.7 is supported
from pipenv.vendor.backports.functools_lru_cache import lru_cache
from .backports.functools import partialmethod # type: ignore
from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size
from .backports.functools import partialmethod # type: ignore
from .backports.surrogateescape import register_surrogateescape
from collections import (
Mapping,
Hashable,
MutableMapping,
Container,
Iterator,
KeysView,
ItemsView,
MappingView,
Iterable,
Set,
Sequence,
Sized,
ValuesView,
MutableSet,
MutableSequence,
Callable,
)
register_surrogateescape()
NamedTemporaryFile = _NamedTemporaryFile
from pipenv.vendor.backports.weakref import finalize # type: ignore
try:
from os.path import samefile
except ImportError:
def samestat(s1, s2):
"""Test whether two stat buffers reference the same file."""
return s1.st_ino == s2.st_ino and s1.st_dev == s2.st_dev
def samefile(f1, f2):
"""Test whether two pathnames reference the same actual file or
directory This is determined by the device number and i-node number
and raises an exception if an os.stat() call on either pathname
fails."""
s1 = os.stat(f1)
s2 = os.stat(f2)
return samestat(s1, s2)
try:
# Introduced Python 3.5
from json import JSONDecodeError
@@ -76,7 +144,7 @@ if six.PY2: # pragma: no cover
pass
class FileNotFoundError(IOError):
"""No such file or directory"""
"""No such file or directory."""
def __init__(self, *args, **kwargs):
self.errno = errno.ENOENT
@@ -95,7 +163,7 @@ if six.PY2: # pragma: no cover
super(TimeoutError, self).__init__(*args, **kwargs)
class IsADirectoryError(OSError):
"""The command does not work on directories"""
"""The command does not work on directories."""
def __init__(self, *args, **kwargs):
self.errno = errno.EISDIR
@@ -118,24 +186,6 @@ else: # pragma: no cover
)
from io import StringIO
six.add_move(
six.MovedAttribute("Iterable", "collections", "collections.abc")
) # type: ignore
six.add_move(
six.MovedAttribute("Mapping", "collections", "collections.abc")
) # type: ignore
six.add_move(
six.MovedAttribute("Sequence", "collections", "collections.abc")
) # type: ignore
six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # type: ignore
six.add_move(
six.MovedAttribute("ItemsView", "collections", "collections.abc")
) # type: ignore
# fmt: off
from six.moves import ItemsView, Iterable, Mapping, Sequence, Set # type: ignore # noqa # isort:skip
# fmt: on
if not sys.warnoptions:
warnings.simplefilter("default", ResourceWarning)
@@ -213,7 +263,7 @@ class TemporaryDirectory(object):
def is_bytes(string):
"""Check if a string is a bytes instance
"""Check if a string is a bytes instance.
:param Union[str, bytes] string: A string that may be string or bytes like
:return: Whether the provided string is a bytes type or not
@@ -227,7 +277,7 @@ def is_bytes(string):
def fs_str(string):
"""Encodes a string into the proper filesystem encoding
"""Encodes a string into the proper filesystem encoding.
Borrowed from pip-tools
"""
@@ -239,8 +289,7 @@ def fs_str(string):
def _get_path(path):
"""
Fetch the string value from a path-like object
"""Fetch the string value from a path-like object.
Returns **None** if there is no string value.
"""
@@ -324,8 +373,7 @@ def _chunks(b, indexes):
def fs_encode(path):
"""
Encode a filesystem path to the proper filesystem encoding
"""Encode a filesystem path to the proper filesystem encoding.
:param Union[str, bytes] path: A string-like path
:returns: A bytes-encoded filesystem path representation
@@ -349,8 +397,7 @@ def fs_encode(path):
def fs_decode(path):
"""
Decode a filesystem path using the proper filesystem encoding
"""Decode a filesystem path using the proper filesystem encoding.
:param path: The filesystem path to decode from bytes or string
:return: The filesystem path, decoded with the determined encoding
@@ -376,17 +423,15 @@ def fs_decode(path):
if sys.version_info[0] < 3: # pragma: no cover
_fs_encode_errors = "surrogateescape"
_fs_encode_errors = "surrogatepass" if sys.platform == "win32" else "surrogateescape"
_fs_decode_errors = "surrogateescape"
_fs_encoding = "utf-8"
else: # pragma: no cover
_fs_encoding = "utf-8"
_fs_decode_errors = "surrogateescape"
if sys.platform.startswith("win"):
_fs_error_fn = None
if sys.version_info[:2] > (3, 4):
alt_strategy = "surrogatepass"
else:
alt_strategy = "surrogateescape"
_fs_encode_errors = "surrogatepass"
else:
if sys.version_info >= (3, 3):
_fs_encoding = sys.getfilesystemencoding()
@@ -394,8 +439,8 @@ else: # pragma: no cover
_fs_encoding = sys.getdefaultencoding()
alt_strategy = "surrogateescape"
_fs_error_fn = getattr(sys, "getfilesystemencodeerrors", None)
_fs_encode_errors = _fs_error_fn() if _fs_error_fn else alt_strategy
_fs_decode_errors = _fs_error_fn() if _fs_error_fn else alt_strategy
_fs_encode_errors = _fs_error_fn() if _fs_error_fn else alt_strategy
_fs_decode_errors = _fs_error_fn() if _fs_error_fn else _fs_decode_errors
_byte = chr if sys.version_info < (3,) else lambda i: bytes([i])
+73 -17
View File
@@ -9,9 +9,34 @@ from contextlib import closing, contextmanager
import six
from .compat import NamedTemporaryFile, Path
from .compat import IS_TYPE_CHECKING, NamedTemporaryFile, Path
from .path import is_file_url, is_valid_url, path_to_url, url_to_path
if IS_TYPE_CHECKING:
from typing import (
Any,
Bytes,
Callable,
ContextManager,
Dict,
IO,
Iterator,
Optional,
Union,
Text,
Tuple,
TypeVar,
)
from types import ModuleType
from requests import Session
from six.moves.http_client import HTTPResponse as Urllib_HTTPResponse
from urllib3.response import HTTPResponse as Urllib3_HTTPResponse
from .spin import VistirSpinner, DummySpinner
TSpinner = Union[VistirSpinner, DummySpinner]
_T = TypeVar("_T")
__all__ = [
"temp_environ",
"temp_path",
@@ -29,6 +54,7 @@ __all__ = [
# See https://github.com/berdario/pew/blob/master/pew/_utils.py#L82
@contextmanager
def temp_environ():
# type: () -> Iterator[None]
"""Allow the ability to set os.environ temporarily"""
environ = dict(os.environ)
try:
@@ -40,17 +66,30 @@ def temp_environ():
@contextmanager
def temp_path():
# type: () -> Iterator[None]
"""A context manager which allows the ability to set sys.path temporarily
>>> path_from_virtualenv = load_path("/path/to/venv/bin/python")
>>> print(sys.path)
['/home/user/.pyenv/versions/3.7.0/bin', '/home/user/.pyenv/versions/3.7.0/lib/python37.zip', '/home/user/.pyenv/versions/3.7.0/lib/python3.7', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/lib-dynload', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/site-packages']
[
'/home/user/.pyenv/versions/3.7.0/bin',
'/home/user/.pyenv/versions/3.7.0/lib/python37.zip',
'/home/user/.pyenv/versions/3.7.0/lib/python3.7',
'/home/user/.pyenv/versions/3.7.0/lib/python3.7/lib-dynload',
'/home/user/.pyenv/versions/3.7.0/lib/python3.7/site-packages'
]
>>> with temp_path():
sys.path = path_from_virtualenv
# Running in the context of the path above
run(["pip", "install", "stuff"])
>>> print(sys.path)
['/home/user/.pyenv/versions/3.7.0/bin', '/home/user/.pyenv/versions/3.7.0/lib/python37.zip', '/home/user/.pyenv/versions/3.7.0/lib/python3.7', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/lib-dynload', '/home/user/.pyenv/versions/3.7.0/lib/python3.7/site-packages']
[
'/home/user/.pyenv/versions/3.7.0/bin',
'/home/user/.pyenv/versions/3.7.0/lib/python37.zip',
'/home/user/.pyenv/versions/3.7.0/lib/python3.7',
'/home/user/.pyenv/versions/3.7.0/lib/python3.7/lib-dynload',
'/home/user/.pyenv/versions/3.7.0/lib/python3.7/site-packages'
]
"""
path = [p for p in sys.path]
@@ -62,6 +101,7 @@ def temp_path():
@contextmanager
def cd(path):
# type: () -> Iterator[None]
"""Context manager to temporarily change working directories
:param str path: The directory to move into
@@ -88,6 +128,7 @@ def cd(path):
@contextmanager
def dummy_spinner(spin_type, text, **kwargs):
# type: (str, str, Any)
class FakeClass(object):
def __init__(self, text=""):
self.text = text
@@ -110,12 +151,13 @@ def dummy_spinner(spin_type, text, **kwargs):
@contextmanager
def spinner(
spinner_name=None,
start_text=None,
handler_map=None,
nospin=False,
write_to_stdout=True,
spinner_name=None, # type: Optional[str]
start_text=None, # type: Optional[str]
handler_map=None, # type: Optional[Dict[str, Callable]]
nospin=False, # type: bool
write_to_stdout=True, # type: bool
):
# type: (...) -> ContextManager[TSpinner]
"""Get a spinner object or a dummy spinner to wrap a context.
:param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"})
@@ -165,6 +207,7 @@ def spinner(
@contextmanager
def atomic_open_for_write(target, binary=False, newline=None, encoding=None):
# type: (str, bool, Optional[str], Optional[str]) -> None
"""Atomically open `target` for writing.
This is based on Lektor's `atomic_open()` utility, but simplified a lot
@@ -173,8 +216,10 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None):
:param str target: Target filename to write
:param bool binary: Whether to open in binary mode, default False
:param str newline: The newline character to use when writing, determined from system if not supplied
:param str encoding: The encoding to use when writing, defaults to system encoding
:param Optional[str] newline: The newline character to use when writing, determined
from system if not supplied.
:param Optional[str] encoding: The encoding to use when writing, defaults to system
encoding.
How this works:
@@ -234,7 +279,10 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None):
delete=False,
)
# set permissions to 0644
os.chmod(f.name, stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
try:
os.chmod(f.name, stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
except OSError:
pass
try:
yield f
except BaseException:
@@ -254,13 +302,19 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None):
@contextmanager
def open_file(link, session=None, stream=True):
def open_file(
link, # type: Union[_T, str]
session=None, # type: Optional[Session]
stream=True, # type: bool
):
# type: (...) -> ContextManager[Union[IO[bytes], Urllib3_HTTPResponse, Urllib_HTTPResponse]]
"""
Open local or remote file for reading.
:type link: pip._internal.index.Link or str
:type session: requests.Session
:param bool stream: Try to stream if remote, default True
:param pip._internal.index.Link link: A link object from resolving dependencies with
pip, or else a URL.
:param Optional[Session] session: A :class:`~requests.Session` instance
:param bool stream: Whether to stream the content if remote, default True
:raises ValueError: If link points to a local directory.
:return: a context manager to the opened file-like object
"""
@@ -286,7 +340,7 @@ def open_file(link, session=None, stream=True):
headers = {"Accept-Encoding": "identity"}
if not session:
try:
from requests import Session
from requests import Session # noqa
except ImportError:
session = None
else:
@@ -302,7 +356,7 @@ def open_file(link, session=None, stream=True):
yield result
finally:
if raw:
conn = getattr(raw, "_connection")
conn = raw._connection
if conn is not None:
conn.close()
result.close()
@@ -310,6 +364,7 @@ def open_file(link, session=None, stream=True):
@contextmanager
def replaced_stream(stream_name):
# type: (str) -> Iterator[IO[Text]]
"""
Context manager to temporarily swap out *stream_name* with a stream wrapper.
@@ -336,6 +391,7 @@ def replaced_stream(stream_name):
@contextmanager
def replaced_streams():
# type: () -> Iterator[Tuple[IO[Text], IO[Text]]]
"""
Context manager to replace both ``sys.stdout`` and ``sys.stderr`` using
``replaced_stream``
+375 -119
View File
@@ -1,19 +1,23 @@
# -*- coding=utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import atexit
import io
import itertools
import json
import locale
import logging
import os
import subprocess
import sys
import threading
from collections import OrderedDict
from functools import partial
from itertools import islice, tee
from weakref import WeakKeyDictionary
import six
from six.moves.queue import Empty, Queue
from .cmdparse import Script
from .compat import (
@@ -21,6 +25,8 @@ from .compat import (
Path,
StringIO,
TimeoutError,
_fs_decode_errors,
_fs_encode_errors,
fs_str,
is_bytes,
partialmethod,
@@ -58,7 +64,7 @@ __all__ = [
if MYPY_RUNNING:
from typing import Any, Dict, List, Optional, Union
from typing import Any, Dict, Generator, IO, List, Optional, Text, Tuple, Union
from .spin import VistirSpinner
@@ -66,8 +72,7 @@ def _get_logger(name=None, level="ERROR"):
# type: (Optional[str], str) -> logging.Logger
if not name:
name = __name__
if isinstance(level, six.string_types):
level = getattr(logging, level.upper())
level = getattr(logging, level.upper())
logger = logging.getLogger(name)
logger.setLevel(level)
formatter = logging.Formatter(
@@ -83,8 +88,9 @@ def shell_escape(cmd):
# type: (Union[str, List[str]]) -> str
"""Escape strings for use in :func:`~subprocess.Popen` and :func:`run`.
This is a passthrough method for instantiating a :class:`~vistir.cmdparse.Script`
object which can be used to escape commands to output as a single string.
This is a passthrough method for instantiating a
:class:`~vistir.cmdparse.Script` object which can be used to escape
commands to output as a single string.
"""
cmd = Script.parse(cmd)
return cmd.cmdify()
@@ -92,14 +98,25 @@ def shell_escape(cmd):
def unnest(elem):
# type: (Iterable) -> Any
"""Flatten an arbitrarily nested iterable
"""Flatten an arbitrarily nested iterable.
:param elem: An iterable to flatten
:type elem: :class:`~collections.Iterable`
>>> nested_iterable = (1234, (3456, 4398345, (234234)), (2396, (23895750, 9283798, 29384, (289375983275, 293759, 2347, (2098, 7987, 27599)))))
>>> nested_iterable = (
1234, (3456, 4398345, (234234)), (
2396, (
23895750, 9283798, 29384, (
289375983275, 293759, 2347, (
2098, 7987, 27599
)
)
)
)
)
>>> list(vistir.misc.unnest(nested_iterable))
[1234, 3456, 4398345, 234234, 2396, 23895750, 9283798, 29384, 289375983275, 293759, 2347, 2098, 7987, 27599]
[1234, 3456, 4398345, 234234, 2396, 23895750, 9283798, 29384, 289375983275, 293759,
2347, 2098, 7987, 27599]
"""
if isinstance(elem, Iterable) and not isinstance(elem, six.string_types):
@@ -127,14 +144,19 @@ def _is_iterable(elem):
def dedup(iterable):
# type: (Iterable) -> Iterable
"""Deduplicate an iterable object like iter(set(iterable)) but
order-reserved.
"""
"""Deduplicate an iterable object like iter(set(iterable)) but order-
preserved."""
return iter(OrderedDict.fromkeys(iterable))
def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=True):
# type: (Union[str, List[str]], Optional[Dict[str, str], bool, Optional[str], bool]) -> subprocess.Popen
def _spawn_subprocess(
script, # type: Union[str, List[str]]
env=None, # type: Optional[Dict[str, str]]
block=True, # type: bool
cwd=None, # type: Optional[Union[str, Path]]
combine_stderr=True, # type: bool
):
# type: (...) -> subprocess.Popen
from distutils.spawn import find_executable
if not env:
@@ -147,6 +169,10 @@ def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=Tru
"stderr": subprocess.PIPE if not combine_stderr else subprocess.STDOUT,
"shell": False,
}
if sys.version_info[:2] > (3, 5):
options.update({"universal_newlines": True, "encoding": "utf-8"})
elif os.name != "nt":
options["universal_newlines"] = True
if not block:
options["stdin"] = subprocess.PIPE
if cwd:
@@ -170,79 +196,295 @@ def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=Tru
return subprocess.Popen(script.cmdify(), **options)
def _read_streams(stream_dict):
results = {}
for outstream in stream_dict.keys():
stream = stream_dict[outstream]
if not stream:
results[outstream] = None
continue
line = to_text(stream.readline())
if not line:
results[outstream] = None
continue
line = to_text("{0}".format(line.rstrip()))
results[outstream] = line
return results
class SubprocessStreamWrapper(object):
def __init__(
self,
display_stderr_maxlen=200, # type: int
display_line_for_loops=20, # type: int
subprocess=None, # type: subprocess.Popen
spinner=None, # type: Optional[VistirSpinner]
verbose=False, # type: bool
stdout_allowed=False, # type: bool
):
# type: (...) -> None
if subprocess is not None:
stdout_encoding = self.get_subprocess_encoding(subprocess, "stdout")
stderr_encoding = self.get_subprocess_encoding(subprocess, "stderr")
self.stdout_encoding = stdout_encoding or PREFERRED_ENCODING
self.stderr_encoding = stderr_encoding or PREFERRED_ENCODING
self.stdout_lines = []
self.text_stdout_lines = []
self.stderr_lines = []
self.text_stderr_lines = []
self.display_line = ""
self.display_line_loops_displayed = 0
self.display_line_shown_for_loops = display_line_for_loops
self.display_line_max_len = display_stderr_maxlen
self.spinner = spinner
self.stdout_allowed = stdout_allowed
self.verbose = verbose
self._iterated_stdout = None
self._iterated_stderr = None
self._subprocess = subprocess
self._queues = {
"streams": Queue(),
"lines": Queue(),
}
self._threads = {
stream_name: threading.Thread(
target=self.enqueue_stream,
args=(self._subprocess, stream_name, self._queues["streams"]),
)
for stream_name in ("stdout", "stderr")
}
self._threads["watcher"] = threading.Thread(
target=self.process_output_lines,
args=(self._queues["streams"], self._queues["lines"]),
)
self.start_threads()
def enqueue_stream(self, proc, stream_name, queue):
# type: (subprocess.Popen, str, Queue) -> None
if not getattr(proc, stream_name, None):
queue.put(("stderr", None))
else:
for line in iter(getattr(proc, stream_name).readline, ""):
queue.put((stream_name, line))
getattr(proc, stream_name).close()
def get_stream_results(cmd_instance, verbose, maxlen, spinner=None, stdout_allowed=False):
stream_results = {"stdout": [], "stderr": []}
streams = {"stderr": cmd_instance.stderr, "stdout": cmd_instance.stdout}
while True:
stream_contents = _read_streams(streams)
stdout_line = stream_contents["stdout"]
stderr_line = stream_contents["stderr"]
if not (stdout_line or stderr_line):
break
last_changed = 0
display_line = ""
for stream_name in stream_contents.keys():
if stream_contents[stream_name] and stream_name in stream_results:
line = stream_contents[stream_name]
stream_results[stream_name].append(line)
display_line = (
fs_str("{0}".format(line))
if stream_name == "stderr"
else display_line
)
if display_line and last_changed > 10:
last_changed = 0
display_line = ""
elif display_line:
last_changed += 1
if len(display_line) > maxlen:
display_line = "{0}...".format(display_line[:maxlen])
@property
def stderr(self):
return self._subprocess.stderr
@property
def stdout(self):
return self._subprocess.stdout
@classmethod
def get_subprocess_encoding(cls, cmd_instance, stream_name):
# type: (subprocess.Popen, str) -> Optional[str]
stream = getattr(cmd_instance, stream_name, None)
if stream is not None:
return get_output_encoding(getattr(stream, "encoding", None))
return None
@property
def stdout_iter(self):
if self._iterated_stdout is None and self.stdout:
self._iterated_stdout = iter(self.stdout.readline, "")
return self._iterated_stdout
@property
def stderr_iter(self):
if self._iterated_stderr is None and self.stderr:
self._iterated_stderr = iter(self.stderr.readline, "")
return self._iterated_stderr
def _decode_line(self, line, encoding):
# type: (Union[str, bytes], str) -> str
if isinstance(line, six.binary_type):
line = to_text(
line.decode(encoding, errors=_fs_decode_errors).encode(
"utf-8", errors=_fs_encode_errors
),
errors="backslashreplace",
)
else:
line = to_text(line, encoding=encoding, errors=_fs_encode_errors)
return line
def start_threads(self):
for thread in self._threads.values():
thread.daemon = True
thread.start()
@property
def subprocess(self):
return self._subprocess
@property
def out(self):
# type: () -> str
return getattr(self.subprocess, "out", "")
@out.setter
def out(self, value):
# type: (str) -> None
self._subprocess.out = value
@property
def err(self):
# type: () -> str
return getattr(self.subprocess, "err", "")
@err.setter
def err(self, value):
# type: (str) -> None
self._subprocess.err = value
def poll(self):
# type: () -> Optional[int]
return self.subprocess.poll()
def wait(self, timeout=None):
# type: (self, Optional[int]) -> Optional[int]
kwargs = {}
if sys.version_info[0] >= 3:
kwargs = {"timeout": timeout}
result = self._subprocess.wait(**kwargs)
self.gather_output()
return result
@property
def returncode(self):
# type: () -> Optional[int]
return self.subprocess.returncode
@property
def text_stdout(self):
return os.linesep.join(self.text_stdout_lines)
@property
def text_stderr(self):
return os.linesep.join(self.text_stderr_lines)
@property
def stderr_closed(self):
# type: () -> bool
return self.stderr is None or (self.stderr is not None and self.stderr.closed)
@property
def stdout_closed(self):
# type: () -> bool
return self.stdout is None or (self.stdout is not None and self.stdout.closed)
@property
def running(self):
# type: () -> bool
return any(t.is_alive() for t in self._threads.values()) or not all(
[self.stderr_closed, self.stdout_closed, self.subprocess_finished]
)
@property
def subprocess_finished(self):
if self._subprocess is None:
return False
return (
self._subprocess.poll() is not None or self._subprocess.returncode is not None
)
def update_display_line(self, new_line):
# type: () -> None
if self.display_line:
if new_line != self.display_line:
self.display_line_loops_displayed = 0
new_line = fs_str("{}".format(new_line))
if len(new_line) > self.display_line_max_len:
new_line = "{}...".format(new_line[: self.display_line_max_len])
self.display_line = new_line
elif self.display_line_loops_displayed >= self.display_line_shown_for_loops:
self.display_line = ""
self.display_line_loops_displayed = 0
else:
self.display_line_loops_displayed += 1
return None
@classmethod
def check_line_content(cls, line):
# type: (Optional[str]) -> bool
return line is not None and line != ""
def get_line(self, queue):
# type: (Queue) -> Tuple[Optional[str], ...]
stream, result = None, None
try:
stream, result = queue.get_nowait()
except Empty:
result = None
return stream, result
def process_output_lines(self, recv_queue, line_queue):
# type: (Queue, Queue) -> None
stream, line = self.get_line(recv_queue)
while self.poll() is None or line is not None:
if self.check_line_content(line):
line = to_text("{}".format(line).rstrip())
line_queue.put((stream, line))
stream, line = self.get_line(recv_queue)
def gather_output(self, spinner=None, stdout_allowed=False, verbose=False):
# type: (Optional[VistirSpinner], bool, bool) -> None
if not getattr(self._subprocess, "out", None):
self._subprocess.out = ""
if not getattr(self._subprocess, "err", None):
self._subprocess.err = ""
if not self._queues["streams"].empty():
self.process_output_lines(self._queues["streams"], self._queues["lines"])
while not self._queues["lines"].empty():
try:
stream_name, line = self._queues["lines"].get()
except Empty:
if not self._threads["watcher"].is_active():
break
pass
if stream_name == "stdout":
text_line = self._decode_line(line, self.stdout_encoding)
self.text_stdout_lines.append(text_line)
self.out += "{}\n".format(text_line)
if verbose:
use_stderr = not stdout_allowed or stream_name != "stdout"
if spinner:
target = spinner.stderr if use_stderr else spinner.stdout
spinner.hide_and_write(display_line, target=target)
else:
target = sys.stderr if use_stderr else sys.stdout
target.write(display_line)
target.flush()
if spinner:
spinner.text = to_native_string(
"{0} {1}".format(spinner.text, display_line)
_write_subprocess_result(
line, "stdout", spinner=spinner, stdout_allowed=stdout_allowed
)
continue
return stream_results
else:
text_err = self._decode_line(line, self.stderr_encoding)
self.text_stderr_lines.append(text_err)
self.update_display_line(line)
self.err += "{}\n".format(text_err)
_write_subprocess_result(
line, "stderr", spinner=spinner, stdout_allowed=stdout_allowed
)
if spinner:
spinner.text = to_native_string(
"{} {}".format(spinner.text, self.display_line)
)
self.out = self.out.strip()
self.err = self.err.strip()
def _write_subprocess_result(result, stream_name, spinner=None, stdout_allowed=False):
# type: (str, str, Optional[VistirSpinner], bool) -> None
if not stdout_allowed and stream_name == "stdout":
stream_name = "stderr"
if spinner:
spinner.hide_and_write(result, target=getattr(spinner, stream_name))
else:
target_stream = getattr(sys, stream_name)
target_stream.write(result)
target_stream.flush()
return None
def attach_stream_reader(
cmd_instance, verbose, maxlen, spinner=None, stdout_allowed=False
):
streams = SubprocessStreamWrapper(
subprocess=cmd_instance,
display_stderr_maxlen=maxlen,
spinner=spinner,
verbose=verbose,
stdout_allowed=stdout_allowed,
)
streams.gather_output(spinner=spinner, verbose=verbose, stdout_allowed=stdout_allowed)
return streams
def _handle_nonblocking_subprocess(c, spinner=None):
# type: (subprocess.Popen, VistirSpinner) -> subprocess.Popen
try:
while c.running:
c.wait()
finally:
if c.stdout:
c.stdout.close()
if c.stderr:
c.stderr.close()
if spinner:
if c.returncode > 0:
if c.returncode != 0:
spinner.fail(to_native_string("Failed...cleaning up..."))
if not os.name == "nt":
elif c.returncode == 0 and not os.name == "nt":
spinner.ok(to_native_string("✔ Complete"))
else:
spinner.ok(to_native_string("Complete"))
@@ -284,7 +526,7 @@ def _create_subprocess(
spinner_orig_text = spinner.text
if not spinner_orig_text and start_text is not None:
spinner_orig_text = start_text
stream_results = get_stream_results(
c = attach_stream_reader(
c,
verbose=verbose,
maxlen=display_limit,
@@ -292,10 +534,6 @@ def _create_subprocess(
stdout_allowed=write_to_stdout,
)
_handle_nonblocking_subprocess(c, spinner)
output = stream_results["stdout"]
err = stream_results["stderr"]
c.out = "\n".join(output) if output else ""
c.err = "\n".join(err) if err else ""
else:
try:
c.out, c.err = c.communicate()
@@ -303,10 +541,6 @@ def _create_subprocess(
c.terminate()
c.out, c.err = c.communicate()
raise
if not block:
c.wait()
c.out = to_text("{0}".format(c.out)) if c.out else fs_str("")
c.err = to_text("{0}".format(c.err)) if c.err else fs_str("")
if not return_object:
return c.out.strip(), c.err.strip()
return c
@@ -330,14 +564,19 @@ def run(
:param list cmd: A list representing the command you want to run.
:param dict env: Additional environment settings to pass through to the subprocess.
:param bool return_object: When True, returns the whole subprocess instance
:param bool block: When False, returns a potentially still-running :class:`subprocess.Popen` instance
:param bool block: When False, returns a potentially still-running
:class:`subprocess.Popen` instance
:param str cwd: Current working directory contect to use for spawning the subprocess.
:param bool verbose: Whether to print stdout in real time when non-blocking.
:param bool nospin: Whether to disable the cli spinner.
:param str spinner_name: The name of the spinner to use if enabled, defaults to bouncingBar
:param bool combine_stderr: Optionally merge stdout and stderr in the subprocess, false if nonblocking.
:param int dispay_limit: The max width of output lines to display when using a spinner.
:param bool write_to_stdout: Whether to write to stdout when using a spinner, default True.
:param str spinner_name: The name of the spinner to use if enabled, defaults to
bouncingBar
:param bool combine_stderr: Optionally merge stdout and stderr in the subprocess,
false if nonblocking.
:param int dispay_limit: The max width of output lines to display when using a
spinner.
:param bool write_to_stdout: Whether to write to stdout when using a spinner,
defaults to True.
:returns: A 2-tuple of (output, error) or a :class:`subprocess.Popen` object.
.. Warning:: Merging standard out and standarad error in a nonblocking subprocess
@@ -346,11 +585,13 @@ def run(
"""
_env = os.environ.copy()
_env["PYTHONIOENCODING"] = str("utf-8")
_env["PYTHONUTF8"] = str("1")
if env:
_env.update(env)
if six.PY2:
fs_encode = partial(to_bytes, encoding=locale_encoding)
_env = {fs_encode(k): fs_encode(v) for k, v in _env.items()}
_fs_encode = partial(to_bytes, encoding=locale_encoding)
_env = {_fs_encode(k): _fs_encode(v) for k, v in _env.items()}
else:
_env = {k: fs_str(v) for k, v in _env.items()}
if not spinner_name:
@@ -386,14 +627,21 @@ def run(
def load_path(python):
"""Load the :mod:`sys.path` from the given python executable's environment as json
"""Load the :mod:`sys.path` from the given python executable's environment
as json.
:param str python: Path to a valid python executable
:return: A python representation of the `sys.path` value of the given python executable.
:return: A python representation of the `sys.path` value of the given python
executable.
:rtype: list
>>> load_path("/home/user/.virtualenvs/requirementslib-5MhGuG3C/bin/python")
['', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python37.zip', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7/lib-dynload', '/home/user/.pyenv/versions/3.7.0/lib/python3.7', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7/site-packages', '/home/user/git/requirementslib/src']
['', '/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python37.zip',
'/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7',
'/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7/lib-dynload',
'/home/user/.pyenv/versions/3.7.0/lib/python3.7',
'/home/user/.virtualenvs/requirementslib-5MhGuG3C/lib/python3.7/site-packages',
'/home/user/git/requirementslib/src']
"""
python = Path(python).as_posix()
@@ -407,7 +655,7 @@ def load_path(python):
def partialclass(cls, *args, **kwargs):
"""Returns a partially instantiated class
"""Returns a partially instantiated class.
:return: A partial class instance
:rtype: cls
@@ -417,7 +665,15 @@ def partialclass(cls, *args, **kwargs):
<class '__main__.Source'>
>>> source(name="pypi")
>>> source.__dict__
mappingproxy({'__module__': '__main__', '__dict__': <attribute '__dict__' of 'Source' objects>, '__weakref__': <attribute '__weakref__' of 'Source' objects>, '__doc__': None, '__init__': functools.partialmethod(<function Source.__init__ at 0x7f23af429bf8>, , url='https://pypi.org/simple')})
mappingproxy({
'__module__': '__main__',
'__dict__': <attribute '__dict__' of 'Source' objects>,
'__weakref__': <attribute '__weakref__' of 'Source' objects>,
'__doc__': None,
'__init__': functools.partialmethod(
<function Source.__init__ at 0x7f23af429bf8>, , url='https://pypi.org/simple'
)
})
>>> new_source = source(name="pypi")
>>> new_source
<__main__.Source object at 0x7f23af189b38>
@@ -526,8 +782,8 @@ def to_text(string, encoding="utf-8", errors=None):
def divide(n, iterable):
"""
split an iterable into n groups, per https://more-itertools.readthedocs.io/en/latest/api.html#grouping
"""split an iterable into n groups, per https://more-
itertools.readthedocs.io/en/latest/api.html#grouping.
:param int n: Number of unique groups
:param iter iterable: An iterable to split up
@@ -578,11 +834,11 @@ except Exception:
def getpreferredencoding():
"""Determine the proper output encoding for terminal rendering"""
"""Determine the proper output encoding for terminal rendering."""
# Borrowed from Invoke
# (see https://github.com/pyinvoke/invoke/blob/93af29d/invoke/runners.py#L881)
_encoding = locale.getpreferredencoding(False)
_encoding = sys.getdefaultencoding() or locale.getpreferredencoding(False)
if six.PY2 and not sys.platform == "win32":
_default_encoding = locale.getdefaultlocale()[1]
if _default_encoding is not None:
@@ -594,8 +850,7 @@ PREFERRED_ENCODING = getpreferredencoding()
def get_output_encoding(source_encoding):
"""
Given a source encoding, determine the preferred output encoding.
"""Given a source encoding, determine the preferred output encoding.
:param str source_encoding: The encoding of the source material.
:returns: The output encoding to decode to.
@@ -630,11 +885,13 @@ def _encode(output, encoding=None, errors=None, translation_map=None):
def decode_for_output(output, target_stream=None, translation_map=None):
"""Given a string, decode it for output to a terminal
"""Given a string, decode it for output to a terminal.
:param str output: A string to print to a terminal
:param target_stream: A stream to write to, we will encode to target this stream if possible.
:param dict translation_map: A mapping of unicode character ordinals to replacement strings.
:param target_stream: A stream to write to, we will encode to target this stream if
possible.
:param dict translation_map: A mapping of unicode character ordinals to replacement
strings.
:return: A re-encoded string using the preferred encoding
:rtype: str
"""
@@ -657,8 +914,7 @@ def decode_for_output(output, target_stream=None, translation_map=None):
def get_canonical_encoding_name(name):
# type: (str) -> str
"""
Given an encoding name, get the canonical name from a codec lookup.
"""Given an encoding name, get the canonical name from a codec lookup.
:param str name: The name of the codec to lookup
:return: The canonical version of the codec name
@@ -696,8 +952,8 @@ def _get_binary_buffer(stream):
def get_wrapped_stream(stream, encoding=None, errors="replace"):
"""
Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream.
"""Given a stream, wrap it in a `StreamWrapper` instance and return the
wrapped stream.
:param stream: A stream instance to wrap
:param str encoding: The encoding to use for the stream
@@ -712,7 +968,7 @@ def get_wrapped_stream(stream, encoding=None, errors="replace"):
if stream is not None and encoding is None:
encoding = "utf-8"
if not encoding:
encoding = get_output_encoding(stream)
encoding = get_output_encoding(getattr(stream, "encoding", None))
else:
encoding = get_canonical_encoding_name(encoding)
return StreamWrapper(stream, encoding, errors, line_buffering=True)
@@ -720,10 +976,8 @@ def get_wrapped_stream(stream, encoding=None, errors="replace"):
class StreamWrapper(io.TextIOWrapper):
"""
This wrapper class will wrap a provided stream and supply an interface
for compatibility.
"""
"""This wrapper class will wrap a provided stream and supply an interface
for compatibility."""
def __init__(self, stream, encoding, errors, line_buffering=True, **kwargs):
self._stream = stream = _StreamProvider(stream)
@@ -907,7 +1161,7 @@ def _cached_stream_lookup(stream_lookup_func, stream_resolution_func):
def get_text_stream(stream="stdout", encoding=None):
"""Retrieve a unicode stream wrapper around **sys.stdout** or **sys.stderr**.
"""Retrieve a utf-8 stream wrapper around **sys.stdout** or **sys.stderr**.
:param str stream: The name of the stream to wrap from the :mod:`sys` module.
:param str encoding: An optional encoding to use.
@@ -959,7 +1213,8 @@ TEXT_STREAMS = {
def replace_with_text_stream(stream_name):
"""Given a stream name, replace the target stream with a text-converted equivalent
"""Given a stream name, replace the target stream with a text-converted
equivalent.
:param str stream_name: The name of a target stream, such as **stdout** or **stderr**
:return: None
@@ -984,7 +1239,8 @@ def _can_use_color(stream=None, color=None):
def echo(text, fg=None, bg=None, style=None, file=None, err=False, color=None):
"""Write the given text to the provided stream or **sys.stdout** by default.
"""Write the given text to the provided stream or **sys.stdout** by
default.
Provides optional foreground and background colors from the ansi defaults:
**grey**, **red**, **green**, **yellow**, **blue**, **magenta**, **cyan**
@@ -1002,7 +1258,7 @@ def echo(text, fg=None, bg=None, style=None, file=None, err=False, color=None):
"""
if file and not hasattr(file, "write"):
raise TypeError("Expected a writable stream, received {0!r}".format(file))
raise TypeError("Expected a writable stream, received {!r}".format(file))
if not file:
if err:
file = _text_stderr()
+97 -77
View File
@@ -8,7 +8,9 @@ import os
import posixpath
import shutil
import stat
import sys
import time
import unicodedata
import warnings
import six
@@ -39,7 +41,27 @@ else:
if IS_TYPE_CHECKING:
from typing import Optional, Callable, Text, ByteString, AnyStr
from types import TracebackType
from typing import (
Any,
AnyStr,
ByteString,
Callable,
Generator,
Iterator,
List,
Optional,
Text,
Tuple,
Type,
Union,
)
if six.PY3:
TPath = os.PathLike
else:
TPath = Union[str, bytes]
TFunc = Callable[..., Any]
__all__ = [
"check_for_unc_path",
@@ -72,16 +94,18 @@ if os.name == "nt":
def unicode_path(path):
# type: (TPath) -> Text
# Paths are supposed to be represented as unicode here
if six.PY2 and not isinstance(path, six.text_type):
if six.PY2 and isinstance(path, six.binary_type):
return path.decode(_fs_encoding)
return path
def native_path(path):
if six.PY2 and not isinstance(path, bytes):
# type: (TPath) -> str
if six.PY2 and isinstance(path, six.text_type):
return path.encode(_fs_encoding)
return path
return str(path)
# once again thank you django...
@@ -91,20 +115,18 @@ if six.PY3 or os.name == "nt":
else:
def abspathu(path):
"""
Version of os.path.abspath that uses the unicode representation
of the current working directory, thus avoiding a UnicodeDecodeError
in join when the cwd has non-ASCII characters.
"""
# type: (TPath) -> Text
"""Version of os.path.abspath that uses the unicode representation of
the current working directory, thus avoiding a UnicodeDecodeError in
join when the cwd has non-ASCII characters."""
if not os.path.isabs(path):
path = os.path.join(os.getcwdu(), path)
return os.path.normpath(path)
def normalize_path(path):
# type: (AnyStr) -> AnyStr
"""
Return a case-normalized absolute variable-expanded path.
# type: (TPath) -> Text
"""Return a case-normalized absolute variable-expanded path.
:param str path: The non-normalized path
:return: A normalized, expanded, case-normalized path
@@ -121,9 +143,8 @@ def normalize_path(path):
def is_in_path(path, parent):
# type: (AnyStr, AnyStr) -> bool
"""
Determine if the provided full path is in the given parent root.
# type: (TPath, TPath) -> bool
"""Determine if the provided full path is in the given parent root.
:param str path: The full path to check the location of.
:param str parent: The parent path to check for membership in
@@ -131,11 +152,11 @@ def is_in_path(path, parent):
:rtype: bool
"""
return normalize_path(str(path)).startswith(normalize_path(str(parent)))
return normalize_path(path).startswith(normalize_path(parent))
def normalize_drive(path):
# type: (str) -> Text
# type: (TPath) -> Text
"""Normalize drive in path so they stay consistent.
This currently only affects local drives on Windows, which can be
@@ -144,8 +165,10 @@ def normalize_drive(path):
"""
from .misc import to_text
if os.name != "nt" or not isinstance(path, six.string_types):
return path
if os.name != "nt" or not (
isinstance(path, six.string_types) or getattr(path, "__fspath__", None)
):
return path # type: ignore
drive, tail = os.path.splitdrive(path)
# Only match (lower cased) local drives (e.g. 'c:'), not UNC mounts.
@@ -156,7 +179,7 @@ def normalize_drive(path):
def path_to_url(path):
# type: (str) -> Text
# type: (TPath) -> Text
"""Convert the supplied local path to a file uri.
:param str path: A string pointing to or representing a local path
@@ -169,7 +192,7 @@ def path_to_url(path):
from .misc import to_bytes
if not path:
return path
return path # type: ignore
normalized_path = Path(normalize_drive(os.path.abspath(path))).as_posix()
if os.name == "nt" and normalized_path[1] == ":":
drive, _, path = normalized_path.partition(":")
@@ -177,18 +200,17 @@ def path_to_url(path):
# XXX: actually part of a surrogate pair, but were just incidentally
# XXX: passed in as a piece of a filename
quoted_path = quote(fs_encode(path))
return fs_decode("file:///{0}:{1}".format(drive, quoted_path))
return fs_decode("file:///{}:{}".format(drive, quoted_path))
# XXX: This is also here to help deal with incidental dangling surrogates
# XXX: on linux, by making sure they are preserved during encoding so that
# XXX: we can urlencode the backslash correctly
bytes_path = to_bytes(normalized_path, errors="backslashreplace")
return fs_decode("file://{0}".format(quote(bytes_path)))
return fs_decode("file://{}".format(quote(bytes_path)))
def url_to_path(url):
# type: (str) -> ByteString
"""
Convert a valid file url to a local filesystem path
# type: (str) -> str
"""Convert a valid file url to a local filesystem path.
Follows logic taken from pip's equivalent function
"""
@@ -204,37 +226,41 @@ def url_to_path(url):
def is_valid_url(url):
"""Checks if a given string is an url"""
# type: (Union[str, bytes]) -> bool
"""Checks if a given string is an url."""
from .misc import to_text
if not url:
return url
return url # type: ignore
pieces = urllib_parse.urlparse(to_text(url))
return all([pieces.scheme, pieces.netloc])
def is_file_url(url):
"""Returns true if the given url is a file url"""
# type: (Any) -> bool
"""Returns true if the given url is a file url."""
from .misc import to_text
if not url:
return False
if not isinstance(url, six.string_types):
try:
url = getattr(url, "url")
url = url.url
except AttributeError:
raise ValueError("Cannot parse url from unknown type: {0!r}".format(url))
raise ValueError("Cannot parse url from unknown type: {!r}".format(url))
url = to_text(url, encoding="utf-8")
return urllib_parse.urlparse(url.lower()).scheme == "file"
def is_readonly_path(fn):
# type: (TPath) -> bool
"""Check if a provided path exists and is readonly.
Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)`
Permissions check is `bool(path.stat & stat.S_IREAD)` or `not
os.access(path, os.W_OK)`
"""
fn = fs_encode(fn)
fn = fs_decode(fs_encode(fn))
if os.path.exists(fn):
file_stat = os.stat(fn).st_mode
return not bool(file_stat & stat.S_IWRITE) or not os.access(fn, os.W_OK)
@@ -242,47 +268,35 @@ def is_readonly_path(fn):
def mkdir_p(newdir, mode=0o777):
"""Recursively creates the target directory and all of its parents if they do not
already exist. Fails silently if they do.
# type: (TPath, int) -> None
"""Recursively creates the target directory and all of its parents if they
do not already exist. Fails silently if they do.
:param str newdir: The directory path to ensure
:raises: OSError if a file is encountered along the way
"""
# http://code.activestate.com/recipes/82465-a-friendly-mkdir/
newdir = fs_encode(newdir)
newdir = fs_decode(fs_encode(newdir))
if os.path.exists(newdir):
if not os.path.isdir(newdir):
raise OSError(
"a file with the same name as the desired dir, '{0}', already exists.".format(
"a file with the same name as the desired dir, '{}', already exists.".format(
fs_decode(newdir)
)
)
else:
head, tail = os.path.split(newdir)
# Make sure the tail doesn't point to the asame place as the head
curdir = fs_encode(".")
tail_and_head_match = (
os.path.relpath(tail, start=os.path.basename(head)) == curdir
)
if tail and not tail_and_head_match and not os.path.isdir(newdir):
target = os.path.join(head, tail)
if os.path.exists(target) and os.path.isfile(target):
raise OSError(
"A file with the same name as the desired dir, '{0}', already exists.".format(
fs_decode(newdir)
)
)
os.makedirs(os.path.join(head, tail), mode)
return None
os.makedirs(newdir, mode)
def ensure_mkdir_p(mode=0o777):
"""Decorator to ensure `mkdir_p` is called to the function's return value.
"""
# type: (int) -> Callable[Callable[..., Any], Callable[..., Any]]
"""Decorator to ensure `mkdir_p` is called to the function's return
value."""
def decorator(f):
# type: (Callable[..., Any]) -> Callable[..., Any]
@functools.wraps(f)
def decorated(*args, **kwargs):
# type: () -> str
path = f(*args, **kwargs)
mkdir_p(path, mode=mode)
return path
@@ -296,6 +310,7 @@ TRACKED_TEMPORARY_DIRECTORIES = []
def create_tracked_tempdir(*args, **kwargs):
# type: (Any, Any) -> str
"""Create a tracked temporary directory.
This uses `TemporaryDirectory`, but does not remove the directory when
@@ -313,6 +328,7 @@ def create_tracked_tempdir(*args, **kwargs):
def create_tracked_tempfile(*args, **kwargs):
# type: (Any, Any) -> str
"""Create a tracked temporary file.
This uses the `NamedTemporaryFile` construct, but does not remove the file
@@ -326,6 +342,7 @@ def create_tracked_tempfile(*args, **kwargs):
def _find_icacls_exe():
# type: () -> Optional[Text]
if os.name == "nt":
paths = [
os.path.expandvars(r"%windir%\{0}").format(subdir)
@@ -343,15 +360,14 @@ def _find_icacls_exe():
def set_write_bit(fn):
# type: (str) -> None
"""
Set read-write permissions for the current user on the target path. Fail silently
if the path doesn't exist.
"""Set read-write permissions for the current user on the target path. Fail
silently if the path doesn't exist.
:param str fn: The target filename or path
:return: None
"""
fn = fs_encode(fn)
fn = fs_decode(fs_encode(fn))
if not os.path.exists(fn):
return
file_stat = os.stat(fn).st_mode
@@ -367,9 +383,9 @@ def set_write_bit(fn):
c = run(
[
icacls_exe,
"''{0}''".format(fn),
"''{}''".format(fn),
"/grant",
"{0}:WD".format(user_sid),
"{}:WD".format(user_sid),
"/T",
"/C",
"/Q",
@@ -396,8 +412,7 @@ def set_write_bit(fn):
def rmtree(directory, ignore_errors=False, onerror=None):
# type: (str, bool, Optional[Callable]) -> None
"""
Stand-in for :func:`~shutil.rmtree` with additional error-handling.
"""Stand-in for :func:`~shutil.rmtree` with additional error-handling.
This version of `rmtree` handles read-only paths, especially in the case of index
files written by certain source control systems.
@@ -411,20 +426,20 @@ def rmtree(directory, ignore_errors=False, onerror=None):
Setting `ignore_errors=True` may cause this to silently fail to delete the path
"""
directory = fs_encode(directory)
directory = fs_decode(fs_encode(directory))
if onerror is None:
onerror = handle_remove_readonly
try:
shutil.rmtree(directory, ignore_errors=ignore_errors, onerror=onerror)
except (IOError, OSError, FileNotFoundError, PermissionError) as exc:
except (IOError, OSError, FileNotFoundError, PermissionError) as exc: # noqa:B014
# Ignore removal failures where the file doesn't exist
if exc.errno != errno.ENOENT:
raise
def _wait_for_files(path): # pragma: no cover
"""
Retry with backoff up to 1 second to delete files from a directory.
# type: (Union[str, TPath]) -> Optional[List[TPath]]
"""Retry with backoff up to 1 second to delete files from a directory.
:param str path: The path to crawl to delete files from
:return: A list of remaining paths or None
@@ -446,7 +461,7 @@ def _wait_for_files(path): # pragma: no cover
except FileNotFoundError as e:
if e.errno == errno.ENOENT:
return
except (OSError, IOError, PermissionError):
except (OSError, IOError, PermissionError): # noqa:B014
time.sleep(timeout)
timeout *= 2
remaining.append(path)
@@ -456,6 +471,7 @@ def _wait_for_files(path): # pragma: no cover
def handle_remove_readonly(func, path, exc):
# type: (Callable[..., str], TPath, Tuple[Type[OSError], OSError, TracebackType]) -> None
"""Error handler for shutil.rmtree.
Windows source repo folders are read-only by default, so this error handler
@@ -480,7 +496,7 @@ def handle_remove_readonly(func, path, exc):
set_write_bit(path)
try:
func(path)
except (
except ( # noqa:B014
OSError,
IOError,
FileNotFoundError,
@@ -503,7 +519,7 @@ def handle_remove_readonly(func, path, exc):
remaining = _wait_for_files(path)
try:
func(path)
except (OSError, IOError, FileNotFoundError, PermissionError) as e:
except (OSError, IOError, FileNotFoundError, PermissionError) as e: # noqa:B014
if e.errno in PERM_ERRORS:
if e.errno != errno.ENOENT: # File still exists
warnings.warn(default_warning_message.format(path), ResourceWarning)
@@ -513,10 +529,12 @@ def handle_remove_readonly(func, path, exc):
def walk_up(bottom):
# type: (Union[TPath, str]) -> Generator[Tuple[str, List[str], List[str]], None, None]
"""Mimic os.walk, but walk 'up' instead of down the directory tree.
From: https://gist.github.com/zdavkeos/1098474
"""
bottom = os.path.realpath(bottom)
bottom = os.path.realpath(str(bottom))
# Get files in current dir.
try:
names = os.listdir(bottom)
@@ -541,7 +559,8 @@ def walk_up(bottom):
def check_for_unc_path(path):
""" Checks to see if a pathlib `Path` object is a unc path or not"""
# type: (Path) -> bool
"""Checks to see if a pathlib `Path` object is a unc path or not."""
if (
os.name == "nt"
and len(path.drive) > 2
@@ -554,6 +573,7 @@ def check_for_unc_path(path):
def get_converted_relative_path(path, relative_to=None):
# type: (TPath, Optional[TPath]) -> str
"""Convert `path` to be relative.
Given a vague relative path, return the path relative to the given
@@ -609,11 +629,11 @@ def get_converted_relative_path(path, relative_to=None):
def safe_expandvars(value):
"""Call os.path.expandvars if value is a string, otherwise do nothing.
"""
# type: (TPath) -> str
"""Call os.path.expandvars if value is a string, otherwise do nothing."""
if isinstance(value, six.string_types):
return os.path.expandvars(value)
return value
return value # type: ignore
class _TrackedTempfileWrapper(_TemporaryFileWrapper, object):
+52 -2
View File
@@ -12,11 +12,31 @@ from io import StringIO
import colorama
import six
from .compat import to_native_string
from .compat import IS_TYPE_CHECKING, to_native_string
from .cursor import hide_cursor, show_cursor
from .misc import decode_for_output, to_text
from .termcolors import COLOR_MAP, COLORS, DISABLE_COLORS, colored
if IS_TYPE_CHECKING:
from typing import (
Any,
Callable,
ContextManager,
Dict,
IO,
Optional,
Text,
Type,
TypeVar,
Union,
)
TSignalMap = Dict[
Type[signal.SIGINT],
Callable[..., int, str, Union["DummySpinner", "VistirSpinner"]],
]
_T = TypeVar("_T", covariant=True)
try:
import yaspin
except ImportError: # pragma: no cover
@@ -66,6 +86,7 @@ decode_output = functools.partial(decode_for_output, translation_map=TRANSLATION
class DummySpinner(object):
def __init__(self, text="", **kwargs):
# type: (str, Any) -> None
if DISABLE_COLORS:
colorama.init()
self.text = to_native_string(decode_output(text)) if text else ""
@@ -108,6 +129,7 @@ class DummySpinner(object):
pass
def fail(self, exitcode=1, text="FAIL"):
# type: (int, str) -> None
if text is not None and text != "None":
if self.write_to_stdout:
self.write(text)
@@ -116,6 +138,7 @@ class DummySpinner(object):
self._close_output_buffer()
def ok(self, text="OK"):
# type: (str) -> int
if text is not None and text != "None":
if self.write_to_stdout:
self.write(text)
@@ -125,6 +148,7 @@ class DummySpinner(object):
return 0
def hide_and_write(self, text, target=None):
# type: (str, Optional[str]) -> None
if not target:
target = self.stdout
if text is None or isinstance(text, six.string_types) and text == "None":
@@ -136,6 +160,7 @@ class DummySpinner(object):
self._show_cursor(target=target)
def write(self, text=None):
# type: (Optional[str]) -> None
if not self.write_to_stdout:
return self.write_err(text)
if text is None or isinstance(text, six.string_types) and text == "None":
@@ -151,6 +176,7 @@ class DummySpinner(object):
stdout.write(CLEAR_LINE)
def write_err(self, text=None):
# type: (Optional[str]) -> None
if text is None or isinstance(text, six.string_types) and text == "None":
pass
text = to_text(text)
@@ -168,10 +194,12 @@ class DummySpinner(object):
@staticmethod
def _hide_cursor(target=None):
# type: (Optional[IO]) -> None
pass
@staticmethod
def _show_cursor(target=None):
# type: (Optional[IO]) -> None
pass
@@ -183,6 +211,7 @@ class VistirSpinner(SpinBase):
"A spinner class for handling spinners on windows and posix."
def __init__(self, *args, **kwargs):
# type: (Any, Any)
"""
Get a spinner object or a dummy spinner to wrap a context.
@@ -196,7 +225,7 @@ class VistirSpinner(SpinBase):
self.handler = handler
colorama.init()
sigmap = {}
sigmap = {} # type: TSignalMap
if handler:
sigmap.update({signal.SIGINT: handler, signal.SIGTERM: handler})
handler_map = kwargs.pop("handler_map", {})
@@ -218,11 +247,15 @@ class VistirSpinner(SpinBase):
self.out_buff = StringIO()
self.write_to_stdout = write_to_stdout
self.is_dummy = bool(yaspin is None)
self._stop_spin = None # type: Optional[threading.Event]
self._hide_spin = None # type: Optional[threading.Event]
self._spin_thread = None # type: Optional[threading.Thread]
super(VistirSpinner, self).__init__(*args, **kwargs)
if DISABLE_COLORS:
colorama.deinit()
def ok(self, text=u"OK", err=False):
# type: (str, bool) -> None
"""Set Ok (success) finalizer to a spinner."""
# Do not display spin text for ok state
self._text = None
@@ -232,6 +265,7 @@ class VistirSpinner(SpinBase):
self._freeze(_text, err=err)
def fail(self, text=u"FAIL", err=False):
# type: (str, bool) -> None
"""Set fail finalizer to a spinner."""
# Do not display spin text for fail state
self._text = None
@@ -241,6 +275,7 @@ class VistirSpinner(SpinBase):
self._freeze(_text, err=err)
def hide_and_write(self, text, target=None):
# type: (str, Optional[str]) -> None
if not target:
target = self.stdout
if text is None or isinstance(text, six.string_types) and text == u"None":
@@ -252,6 +287,7 @@ class VistirSpinner(SpinBase):
self._show_cursor(target=target)
def write(self, text): # pragma: no cover
# type: (str) -> None
if not self.write_to_stdout:
return self.write_err(text)
stdout = self.stdout
@@ -266,6 +302,7 @@ class VistirSpinner(SpinBase):
self.out_buff.write(text)
def write_err(self, text): # pragma: no cover
# type: (str) -> None
"""Write error text in the terminal without breaking the spinner."""
stderr = self.stderr
if self.stderr.closed:
@@ -279,6 +316,7 @@ class VistirSpinner(SpinBase):
self.out_buff.write(decode_output(text, target_stream=self.out_buff))
def start(self):
# type: () -> None
if self._sigmap:
self._register_signal_handlers()
@@ -292,6 +330,7 @@ class VistirSpinner(SpinBase):
self._spin_thread.start()
def stop(self):
# type: () -> None
if self._dfl_sigmap:
# Reset registered signal handlers to default ones
self._reset_signal_handlers()
@@ -314,6 +353,7 @@ class VistirSpinner(SpinBase):
self.out_buff.close()
def _freeze(self, final_text, err=False):
# type: (str, bool) -> None
"""Stop spinner, compose last frame and 'freeze' it."""
if not final_text:
final_text = ""
@@ -330,12 +370,14 @@ class VistirSpinner(SpinBase):
target.write(self._last_frame)
def _compose_color_func(self):
# type: () -> Callable[..., str]
fn = functools.partial(
colored, color=self._color, on_color=self._on_color, attrs=list(self._attrs)
)
return fn
def _compose_out(self, frame, mode=None):
# type: (str, Optional[str]) -> Text
# Ensure Unicode input
frame = to_text(frame)
@@ -355,6 +397,7 @@ class VistirSpinner(SpinBase):
return out
def _spin(self):
# type: () -> None
target = self.stdout if self.write_to_stdout else self.stderr
clear_fn = self._clear_line if self.write_to_stdout else self._clear_err
while not self._stop_spin.is_set():
@@ -379,6 +422,7 @@ class VistirSpinner(SpinBase):
target.write("\b")
def _register_signal_handlers(self):
# type: () -> None
# SIGKILL cannot be caught or ignored, and the receiving
# process cannot perform any clean-up upon receiving this
# signal.
@@ -411,31 +455,37 @@ class VistirSpinner(SpinBase):
signal.signal(sig, sig_handler)
def _reset_signal_handlers(self):
# type: () -> None
for sig, sig_handler in self._dfl_sigmap.items():
signal.signal(sig, sig_handler)
@staticmethod
def _hide_cursor(target=None):
# type: (Optional[IO]) -> None
if not target:
target = sys.stdout
hide_cursor(stream=target)
@staticmethod
def _show_cursor(target=None):
# type: (Optional[IO]) -> None
if not target:
target = sys.stdout
show_cursor(stream=target)
@staticmethod
def _clear_err():
# type: () -> None
sys.stderr.write(CLEAR_LINE)
@staticmethod
def _clear_line():
# type: () -> None
sys.stdout.write(CLEAR_LINE)
def create_spinner(*args, **kwargs):
# type: (Any, Any) -> Union[DummySpinner, VistirSpinner]
nospin = kwargs.pop("nospin", False)
use_yaspin = kwargs.pop("use_yaspin", not nospin)
if nospin or not use_yaspin:
+1 -1
View File
@@ -432,7 +432,7 @@ def test_system_and_deploy_work(PipenvInstance):
Path(p.pipfile_path).write_text(
u"""
[packages]
requests
requests = "*"
""".strip()
)
c = p.pipenv("install --system")
-2
View File
@@ -97,8 +97,6 @@ setup(
c = pipenv_instance.pipenv("install -v -e .")
assert c.return_code == 0
assert "test-private-dependency" in pipenv_instance.lockfile["default"]
assert "version" in pipenv_instance.lockfile["default"]["test-private-dependency"]
assert "0.1" in pipenv_instance.lockfile["default"]["test-private-dependency"]["version"]
def test_https_dependency_links_install(self, PipenvInstance):
"""Ensure dependency_links are parsed and installed (needed for private repo dependencies).
+1 -3
View File
@@ -13,7 +13,7 @@ from pipenv._compat import Path
@pytest.mark.vcs
@pytest.mark.install
@pytest.mark.needs_internet
def test_basic_vcs_install(PipenvInstance): # ! This is failing
def test_basic_vcs_install(PipenvInstance):
with PipenvInstance(chdir=True) as p:
c = p.pipenv("install git+https://github.com/benjaminp/six.git@1.11.0#egg=six")
assert c.return_code == 0
@@ -25,7 +25,6 @@ def test_basic_vcs_install(PipenvInstance): # ! This is failing
assert p.lockfile["default"]["six"] == {
"git": "https://github.com/benjaminp/six.git",
"ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a",
"version": "==1.11.0"
}
assert "gitdb2" in p.lockfile["default"]
@@ -43,7 +42,6 @@ def test_git_vcs_install(PipenvInstance):
assert p.lockfile["default"]["six"] == {
"git": "git://github.com/benjaminp/six.git",
"ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a",
"version": "==1.11.0"
}
+20
View File
@@ -2,6 +2,7 @@
import json
import os
import shutil
import sys
import pytest
@@ -10,6 +11,7 @@ from flaky import flaky
from vistir.compat import Path
from vistir.misc import to_text
from pipenv.utils import temp_environ
import delegator
@pytest.mark.lock
@@ -728,3 +730,21 @@ def test_lock_nested_direct_url(PipenvInstance):
assert "vistir" in p.lockfile["default"]
assert "colorama" in p.lockfile["default"]
assert "six" in p.lockfile["default"]
@pytest.mark.lock
@pytest.mark.needs_internet
def test_lock_nested_vcs_direct_url(PipenvInstance):
with PipenvInstance(chdir=True) as p:
p._pipfile.add("pep508_package", {
"git": "https://github.com/techalchemy/test-project.git",
"editable": True, "ref": "master",
"subdirectory": "parent_folder/pep508-package"
})
c = p.pipenv("install")
assert c.return_code == 0
assert "git" in p.lockfile["default"]["pep508-package"]
assert "sibling-package" in p.lockfile["default"]
assert "git" in p.lockfile["default"]["sibling-package"]
assert "subdirectory" in p.lockfile["default"]["sibling-package"]
assert "version" not in p.lockfile["default"]["sibling-package"]