This commit is contained in:
2019-04-20 09:12:54 -04:00
parent a79a4f30e1
commit a200e34236
17 changed files with 364 additions and 216 deletions
+92 -4
View File
@@ -114,10 +114,9 @@ from .__version__ import __build__, __author__, __author_email__, __license__
from .__version__ import __copyright__, __cake__
from . import utils
from .models import Request, Response, PreparedRequest
from .api import request, get, head, post, patch, put, delete, options
from .sessions import Session, AsyncSession
from .status_codes import codes
from .http_models import Request, Response, PreparedRequest
from .http_sessions import Session, AsyncSession
from .http_stati import codes
from .exceptions import (
RequestException,
Timeout,
@@ -137,3 +136,92 @@ from logging import NullHandler
logging.getLogger(__name__).addHandler(NullHandler())
# FileModeWarnings go off per the default.
warnings.simplefilter("default", FileModeWarning, append=True)
# -*- coding: utf-8 -*-
"""
requests.api
~~~~~~~~~~~~
This module implements the Requests API.
:copyright: (c) 2012 by Kenneth Reitz.
:license: Apache2, see LICENSE for more details.
"""
from . import http_sessions as sessions
from . import _types as types
def request(
method: types.Method, url: types.URL, *, session: types.Session = None, **kwargs
) -> types.Response:
"""Constructs and sends a :class:`Request <Request>`.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param session: :class:`Session` object to use for this request. If none is given, one will be provided.
:param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
:param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`.
:param json: (optional) json data to send in the body of the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload.
``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')``
or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string
defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers
to add for the file.
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) How many seconds to wait for the server to send data
before giving up, as a float, or a :ref:`(connect timeout, read
timeout) <timeouts>` tuple.
:type timeout: float or tuple
:param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``True``.
:type allow_redirects: bool
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
:param verify: (optional) Either a boolean, in which case it controls whether we verify
the server's TLS certificate, or a string, in which case it must be a path
to a CA bundle to use. Defaults to ``True``.
:param stream: (optional) if ``False``, the response content will be immediately downloaded.
:param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
:return: :class:`Response <Response>` object
:rtype: requests.Response
Usage::
>>> import requests
>>> req = requests.request('GET', 'https://httpbin.org/get')
<Response [200]>
"""
# By using the 'with' statement we are sure the session is closed, thus we
# avoid leaving sockets open which can trigger a ResourceWarning in some
# cases, and look like a memory leak in others.
session = sessions.Session() if session is None else session
with session:
return session.request(method=method, url=url, **kwargs)
def get(url: types.URL, *, params: types.Params = None, **kwargs) -> types.Response:
r"""Sends a GET request.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary, list of tuples or bytes to send
in the query string for the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
kwargs.setdefault("allow_redirects", True)
return request("get", url, params=params, **kwargs)
def head(url: types.URL, **kwargs) -> types.Response:
r"""Sends a HEAD request.
:param url: URL for the new :class:`Request` object.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
kwargs.setdefault("allow_redirects", False)
return request("head", url, **kwargs)
+1 -1
View File
@@ -7,7 +7,7 @@ Provides utility functions that are consumed internally by Requests
which depend on extremely few external helpers (such as compat)
"""
from .basics import builtin_str, str
from ._basics import builtin_str, str
def to_native_string(string, encoding='ascii'):
@@ -8,7 +8,7 @@ Data structures that power Requests.
import collections
from .basics import basestring, OrderedDict
from ._basics import basestring, OrderedDict
class CaseInsensitiveDict(collections.MutableMapping):
@@ -0,0 +1,229 @@
# -*- coding: utf-8 -*-
"""
requests.structures
~~~~~~~~~~~~~~~~~~~
Data structures that power Requests.
"""
import collections
from ._basics import basestring, OrderedDict
class CaseInsensitiveDict(collections.MutableMapping):
"""A case-insensitive ``dict``-like object.
Implements all methods and operations of
``collections.MutableMapping`` as well as dict's ``copy``. Also
provides ``lower_items``.
All keys are expected to be strings. The structure remembers the
case of the last key to be set, and ``iter(instance)``,
``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()``
will contain case-sensitive keys. However, querying and contains
testing is case insensitive::
cid = CaseInsensitiveDict()
cid['Accept'] = 'application/json'
cid['aCCEPT'] == 'application/json' # True
list(cid) == ['Accept'] # True
For example, ``headers['content-encoding']`` will return the
value of a ``'Content-Encoding'`` response header, regardless
of how the header name was originally stored.
If the constructor, ``.update``, or equality comparison
operations are given keys that have equal ``.lower()``s, the
behavior is undefined.
"""
__slots__ = ('_store')
def __init__(self, data=None, **kwargs):
self._store = collections.OrderedDict()
if data is None:
data = {}
self.update(data, **kwargs)
def __setitem__(self, key, value):
# Use the lowercased key for lookups, but store the actual
# key alongside the value.
self._store[key.lower()] = (key, value)
def __getitem__(self, key):
return self._store[key.lower()][1]
def __delitem__(self, key):
del self._store[key.lower()]
def __iter__(self):
return (casedkey for casedkey, mappedvalue in self._store.values())
def __len__(self):
return len(self._store)
def lower_items(self):
"""Like iteritems(), but with all lowercase keys."""
return (
(lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items()
)
def __eq__(self, other):
if isinstance(other, collections.Mapping):
other = CaseInsensitiveDict(other)
else:
return NotImplemented
# Compare insensitively
return dict(self.lower_items()) == dict(other.lower_items())
# Copy is required
def copy(self):
return CaseInsensitiveDict(self._store.values())
def __repr__(self):
return str(dict(self.items()))
class HTTPHeaderDict(CaseInsensitiveDict):
"""A case-insensitive ``dict``-like object suitable for HTTP headers that
supports multiple values with the same key, via the ``add``, ``extend``,
``multiget`` and ``multiset`` methods.
"""
def __init__(self, data=None, **kwargs):
super(HTTPHeaderDict, self).__init__()
self.extend({} if data is None else data, **kwargs)
# We'll store tuples in the internal dictionary, but present them as a
# concatenated string when we use item access methods.
#
def __setitem__(self, key, val):
# Specialcase null values.
if (not isinstance(val, basestring)) and (val is not None):
raise ValueError('only string-type values (or None) are allowed')
super(HTTPHeaderDict, self).__setitem__(key, (val,))
def __getitem__(self, key):
val = super(HTTPHeaderDict, self).__getitem__(key)
# Specialcase null values.
if len(val) == 1 and val[0] is None:
return val[0]
return ', '.join(val)
def lower_items(self):
return (
(lk, ', '.join(vals)) for (lk, (k, vals)) in self._store.items()
)
def copy(self):
return type(self)(self)
def getlist(self, key):
"""Returns a list of all the values for the named field. Returns an
empty list if the key isn't present in the dictionary."""
return list(self._store.get(key.lower(), (None, []))[1])
def setlist(self, key, values):
"""Set a sequence of strings to the associated key - this will overwrite
any previously stored value."""
if not isinstance(values, (list, tuple)):
raise ValueError('argument is not sequence')
if any(not isinstance(v, basestring) for v in values):
raise ValueError('non-string items in sequence')
if not values:
self.pop(key, None)
return
super(HTTPHeaderDict, self).__setitem__(key, tuple(values))
def _extend(self, key, values):
new_value_tpl = key, values
# Inspired by urllib3's implementation - use one call which should be
# suitable for the common case.
old_value_tpl = self._store.setdefault(key.lower(), new_value_tpl)
if old_value_tpl is not new_value_tpl:
old_key, old_values = old_value_tpl
self._store[key.lower()] = (old_key, old_values + values)
def add(self, key, val):
"""Adds a key, value pair to this dictionary - if there is already a
value for this key, then the value will be appended to those values.
"""
if not isinstance(val, basestring):
raise ValueError('value must be a string-type object')
self._extend(key, (val,))
def extend(self, *args, **kwargs):
"""Like update, but will add values to existing sequences rather than
replacing them. You can pass a mapping object or a sequence of two
tuples - values in these objects can be strings or sequence of strings.
"""
if len(args) > 1:
raise TypeError(
f"extend() takes at most 1 positional "
"arguments ({len(args)} given)"
)
for other in args + (kwargs,):
if isinstance(other, collections.Mapping):
# See if looks like a HTTPHeaderDict (either urllib3's
# implementation or ours). If so, then we have to add values
# in one go for each key.
multiget = getattr(other, 'getlist', None)
if multiget:
for key in other:
self._extend(key, tuple(multiget(key)))
continue
# Otherwise, just walk over items to get them.
item_seq = other.items()
else:
item_seq = other
for ik, iv in item_seq:
if isinstance(iv, basestring):
self._extend(ik, (iv,))
elif any(not isinstance(v, basestring) for v in iv):
raise ValueError('non-string items in sequence')
else:
self._extend(ik, tuple(iv))
@property
def _as_dict(self):
"""A dictionary representation of the HTTPHeaderDict."""
d = {}
for k, vals in self._store.values():
d[k] = vals[0] if len(vals) == 1 else vals
return d
def __repr__(self):
return repr(self._as_dict)
class LookupDict(dict):
"""Dictionary lookup object."""
def __init__(self, name=None):
self.name = name
super(LookupDict, self).__init__()
def __repr__(self):
return f'<lookup \'{self.name}\'>'
def __getitem__(self, key):
# We allow fall-through here, so values default to None
return self.__dict__.get(key, None)
def __iter__(self):
return super(LookupDict, self).__dir__()
def get(self, key, default=None):
return self.__dict__.get(key, default)
+4 -4
View File
@@ -14,10 +14,10 @@ from typing import (
Dict,
)
from .import auth
from .models import Response, PreparedRequest
from .cookies import RequestsCookieJar
from .sessions import Session
from .import http_auth as auth
from .http_models import Response, PreparedRequest
from .http_cookies import RequestsCookieJar
from .http_sessions import Session
_ParamsMappingValueType = Union[
str, bytes, int, float, Iterable[Union[str, bytes, int, float]]
-154
View File
@@ -1,154 +0,0 @@
# -*- coding: utf-8 -*-
"""
requests.api
~~~~~~~~~~~~
This module implements the Requests API.
:copyright: (c) 2012 by Kenneth Reitz.
:license: Apache2, see LICENSE for more details.
"""
from . import sessions
from . import types
def request(
method: types.Method, url: types.URL, *, session: types.Session = None, **kwargs
) -> types.Response:
"""Constructs and sends a :class:`Request <Request>`.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param session: :class:`Session` object to use for this request. If none is given, one will be provided.
:param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
:param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`.
:param json: (optional) json data to send in the body of the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload.
``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')``
or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string
defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers
to add for the file.
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) How many seconds to wait for the server to send data
before giving up, as a float, or a :ref:`(connect timeout, read
timeout) <timeouts>` tuple.
:type timeout: float or tuple
:param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``True``.
:type allow_redirects: bool
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
:param verify: (optional) Either a boolean, in which case it controls whether we verify
the server's TLS certificate, or a string, in which case it must be a path
to a CA bundle to use. Defaults to ``True``.
:param stream: (optional) if ``False``, the response content will be immediately downloaded.
:param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
:return: :class:`Response <Response>` object
:rtype: requests.Response
Usage::
>>> import requests
>>> req = requests.request('GET', 'https://httpbin.org/get')
<Response [200]>
"""
# By using the 'with' statement we are sure the session is closed, thus we
# avoid leaving sockets open which can trigger a ResourceWarning in some
# cases, and look like a memory leak in others.
session = sessions.Session() if session is None else session
with session:
return session.request(method=method, url=url, **kwargs)
def get(url: types.URL, *, params: types.Params = None, **kwargs) -> types.Response:
r"""Sends a GET request.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary, list of tuples or bytes to send
in the query string for the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
kwargs.setdefault("allow_redirects", True)
return request("get", url, params=params, **kwargs)
def options(url: types.URL, **kwargs) -> types.Response:
r"""Sends an OPTIONS request.
:param url: URL for the new :class:`Request` object.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
kwargs.setdefault("allow_redirects", True)
return request("options", url, **kwargs)
def head(url: types.URL, **kwargs) -> types.Response:
r"""Sends a HEAD request.
:param url: URL for the new :class:`Request` object.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
kwargs.setdefault("allow_redirects", False)
return request("head", url, **kwargs)
def post(
url: types.URL, *, data: types.Data = None, json: types.JSON = None, **kwargs
) -> types.Response:
r"""Sends a POST request.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param json: (optional) json data to send in the body of the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request("post", url, data=data, json=json, **kwargs)
def put(url: types.URL, *, data: types.Data = None, **kwargs) -> types.Response:
r"""Sends a PUT request.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param json: (optional) json data to send in the body of the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request("put", url, data=data, **kwargs)
def patch(url: types.URL, *, data: types.Data = None, **kwargs) -> types.Response:
r"""Sends a PATCH request.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param json: (optional) json data to send in the body of the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request("patch", url, data=data, **kwargs)
def delete(url: types.URL, **kwargs) -> types.Response:
r"""Sends a DELETE request.
:param url: URL for the new :class:`Request` object.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request("delete", url, **kwargs)
-17
View File
@@ -1,17 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
requests.certs
~~~~~~~~~~~~~~
This module returns the preferred default CA certificate bundle. There is
only one — the one from the certifi package.
If you are packaging Requests, e.g., for a Linux distribution or a managed
environment, you can change the definition of where() to return a separately
packaged CA bundle.
"""
from certifi import where
if __name__ == '__main__':
print(where())
@@ -28,8 +28,8 @@ from .core._http.exceptions import ReadTimeoutError
from .core._http.exceptions import SSLError as _SSLError
from .core._http.exceptions import ResponseError
from .models import Response, AsyncResponse
from .basics import urlparse, basestring
from .http_models import Response, AsyncResponse
from ._basics import urlparse, basestring
from .utils import (
DEFAULT_CA_BUNDLE_PATH,
get_encoding_from_headers,
@@ -38,8 +38,8 @@ from .utils import (
urldefragauth,
select_proxy,
)
from .structures import HTTPHeaderDict
from .cookies import extract_cookies_to_jar
from ._structures import HTTPHeaderDict
from .http_cookies import extract_cookies_to_jar
from .exceptions import (
ConnectionError,
ConnectTimeout,
@@ -49,7 +49,7 @@ from .exceptions import (
RetryError,
InvalidScheme,
)
from .auth import _basic_auth_str
from .http_auth import _basic_auth_str
try:
from .core._http.contrib.socks import SOCKSProxyManager
+2 -2
View File
@@ -14,8 +14,8 @@ import threading
from base64 import b64encode
from .basics import urlparse, str, basestring
from .cookies import extract_cookies_to_jar
from ._basics import urlparse, str, basestring
from .http_cookies import extract_cookies_to_jar
from ._internal_utils import to_native_string
from .utils import parse_dict_header
@@ -14,7 +14,7 @@ import calendar
from collections.abc import MutableMapping
from ._internal_utils import to_native_string
from .basics import cookielib, urlparse, urlunparse, Morsel
from ._basics import cookielib, urlparse, urlunparse, Morsel
try:
import threading
@@ -24,12 +24,12 @@ from .core._http.exceptions import (
)
from io import UnsupportedOperation
from .hooks import default_hooks
from .structures import CaseInsensitiveDict
from ._hooks import default_hooks
from ._structures import CaseInsensitiveDict
import requests3 as requests
from .auth import HTTPBasicAuth
from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar
from .http_auth import HTTPBasicAuth
from .http_cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar
from .exceptions import (
HTTPError,
MissingScheme,
@@ -56,7 +56,7 @@ from .utils import (
check_header_validity,
is_stream,
)
from .basics import (
from ._basics import (
cookielib,
urlunparse,
urlsplit,
@@ -68,7 +68,7 @@ from .basics import (
basestring,
)
import json as complexjson
from .status_codes import codes
from .http_stati import codes
# : The set of HTTP status codes that indicate an automatically
#: processable redirect.
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
requests.session
~~~~~~~~~~~~~~~~
requests.http_session
~~~~~~~~~~~~~~~~~~~~~
This module provides a Session object to manage and persist settings across
requests (cookies, auth, proxies).
@@ -14,17 +14,17 @@ from datetime import timedelta
from .core._http._backends.trio_backend import TrioBackend
from .auth import _basic_auth_str
from .basics import cookielib, urljoin, urlparse, str
from .cookies import (
from .http_auth import _basic_auth_str
from ._basics import cookielib, urljoin, urlparse, str
from .http_cookies import (
cookiejar_from_dict,
extract_cookies_to_jar,
RequestsCookieJar,
merge_cookies,
_copy_cookie_jar,
)
from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT
from .hooks import default_hooks, dispatch_hook
from .http_models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT
from ._hooks import default_hooks, dispatch_hook
from ._internal_utils import to_native_string
from .utils import to_key_val_list, default_headers, DEFAULT_PORTS
from .exceptions import (
@@ -36,8 +36,8 @@ from .exceptions import (
InvalidHeader,
)
from .structures import CaseInsensitiveDict
from .adapters import HTTPAdapter, AsyncHTTPAdapter
from ._structures import CaseInsensitiveDict
from .http_adapters import HTTPAdapter, AsyncHTTPAdapter
from .utils import (
requote_uri,
@@ -49,10 +49,10 @@ from .utils import (
rewind_body,
)
from .status_codes import codes
from .http_stati import codes
# formerly defined here, reexposed here for backward compatibility
from .models import REDIRECT_STATI
from .http_models import REDIRECT_STATI
# Preferred clock, based on which one is more accurate on a given system.
if sys.platform == "win32":
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
from .structures import LookupDict
from ._structures import LookupDict
_codes = {
# Informational.
+9 -8
View File
@@ -23,11 +23,12 @@ import platform
import zipfile
from typing import Mapping
from .__version__ import __version__
from . import certs
import certifi
from .basics import parse_http_list as _parse_list_header
from .basics import (
from .__version__ import __version__
from ._basics import parse_http_list as _parse_list_header
from ._basics import (
quote,
urlparse,
bytes,
@@ -41,9 +42,9 @@ from .basics import (
proxy_bypass_environment,
getproxies_environment
)
from .cookies import cookiejar_from_dict
from .structures import HTTPHeaderDict
from .cookies import RequestsCookieJar
from .http_cookies import cookiejar_from_dict, RequestsCookieJar
from ._structures import HTTPHeaderDict
from .exceptions import (
InvalidURL,
InvalidHeader,
@@ -52,7 +53,7 @@ from .exceptions import (
)
NETRC_FILES = (".netrc", "_netrc")
DEFAULT_CA_BUNDLE_PATH = certs.where()
DEFAULT_CA_BUNDLE_PATH = certifi.where()
DEFAULT_PORTS = (80, 443)
if platform.system() == "Windows":
+2 -1
View File
@@ -73,7 +73,8 @@ requires = [
'chardet>=3.0.2,<3.1.0',
'idna>=2.5,<2.9',
'urllib3>=1.21.1,<1.25',
'certifi>=2017.4.17'
'certifi>=2017.4.17',
'requests-toolbelt'
]
test_requirements = [