kennethreitz/requests
1
0
Fork 0
mirror of https://github.com/kennethreitz/requests.git synced 2026-06-05 14:50:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

black

Browse Source 
https://github.com/ambv/black
This commit is contained in:
Kenneth Reitz
2018-03-14 17:28:51 -04:00
parent 2bf628be3c
commit 9cea8ce09d
33 changed files with 1422 additions and 1612 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Pipfile
+4 -2
View File
@@ -22,9 +22,11 @@ tox = "*"
detox = "*"
httpbin = "==0.5.0"
pytest-mypy = "*"
black = {git = "https://github.com/ambv/black.git", editable = true}
"e1839a8" = {path = ".", editable = true, extras=["socks"]}
mypy = "==0.540"
win-inet-ptonsocks = {version="*", os_name = "=='windows'"}
[packages]
"e1839a8" = {path = ".", editable = true, extras=["socks"]}
mypy = "==0.540"
docs/_themes/flask_theme_support.py
Vendored
+62 -72
View File
@@ -1,86 +1,76 @@
# flasky extensions. flasky pygments style based on tango style
from pygments.style import Style
from pygments.token import Keyword, Name, Comment, String, Error, \
Number, Operator, Generic, Whitespace, Punctuation, Other, Literal
from pygments.token import Keyword, Name, Comment, String, Error, Number, Operator, Generic, Whitespace, Punctuation, Other, Literal
class FlaskyStyle(Style):
background_color = "#f8f8f8"
default_style = ""
styles = {
# No corresponding class for the following:
#Text: "", # class: ''
Whitespace: "underline #f8f8f8", # class: 'w'
Error: "#a40000 border:#ef2929", # class: 'err'
Other: "#000000", # class 'x'
Comment: "italic #8f5902", # class: 'c'
Comment.Preproc: "noitalic", # class: 'cp'
Keyword: "bold #004461", # class: 'k'
Keyword.Constant: "bold #004461", # class: 'kc'
Keyword.Declaration: "bold #004461", # class: 'kd'
Keyword.Namespace: "bold #004461", # class: 'kn'
Keyword.Pseudo: "bold #004461", # class: 'kp'
Keyword.Reserved: "bold #004461", # class: 'kr'
Keyword.Type: "bold #004461", # class: 'kt'
Operator: "#582800", # class: 'o'
Operator.Word: "bold #004461", # class: 'ow' - like keywords
Punctuation: "bold #000000", # class: 'p'
Whitespace: "underline #f8f8f8", # class: 'w'
Error: "#a40000 border:#ef2929", # class: 'err'
Other: "#000000", # class 'x'
Comment: "italic #8f5902", # class: 'c'
Comment.Preproc: "noitalic", # class: 'cp'
Keyword: "bold #004461", # class: 'k'
Keyword.Constant: "bold #004461", # class: 'kc'
Keyword.Declaration: "bold #004461", # class: 'kd'
Keyword.Namespace: "bold #004461", # class: 'kn'
Keyword.Pseudo: "bold #004461", # class: 'kp'
Keyword.Reserved: "bold #004461", # class: 'kr'
Keyword.Type: "bold #004461", # class: 'kt'
Operator: "#582800", # class: 'o'
Operator.Word: "bold #004461", # class: 'ow' - like keywords
Punctuation: "bold #000000",
# class: 'p'
# because special names such as Name.Class, Name.Function, etc.
# are not recognized as such later in the parsing, we choose them
# to look the same as ordinary variables.
Name: "#000000", # class: 'n'
Name.Attribute: "#c4a000", # class: 'na' - to be revised
Name.Builtin: "#004461", # class: 'nb'
Name.Builtin.Pseudo: "#3465a4", # class: 'bp'
Name.Class: "#000000", # class: 'nc' - to be revised
Name.Constant: "#000000", # class: 'no' - to be revised
Name.Decorator: "#888", # class: 'nd' - to be revised
Name.Entity: "#ce5c00", # class: 'ni'
Name.Exception: "bold #cc0000", # class: 'ne'
Name.Function: "#000000", # class: 'nf'
Name.Property: "#000000", # class: 'py'
Name.Label: "#f57900", # class: 'nl'
Name.Namespace: "#000000", # class: 'nn' - to be revised
Name.Other: "#000000", # class: 'nx'
Name.Tag: "bold #004461", # class: 'nt' - like a keyword
Name.Variable: "#000000", # class: 'nv' - to be revised
Name.Variable.Class: "#000000", # class: 'vc' - to be revised
Name.Variable.Global: "#000000", # class: 'vg' - to be revised
Name.Variable.Instance: "#000000", # class: 'vi' - to be revised
Number: "#990000", # class: 'm'
Literal: "#000000", # class: 'l'
Literal.Date: "#000000", # class: 'ld'
String: "#4e9a06", # class: 's'
String.Backtick: "#4e9a06", # class: 'sb'
String.Char: "#4e9a06", # class: 'sc'
String.Doc: "italic #8f5902", # class: 'sd' - like a comment
String.Double: "#4e9a06", # class: 's2'
String.Escape: "#4e9a06", # class: 'se'
String.Heredoc: "#4e9a06", # class: 'sh'
String.Interpol: "#4e9a06", # class: 'si'
String.Other: "#4e9a06", # class: 'sx'
String.Regex: "#4e9a06", # class: 'sr'
String.Single: "#4e9a06", # class: 's1'
String.Symbol: "#4e9a06", # class: 'ss'
Generic: "#000000", # class: 'g'
Generic.Deleted: "#a40000", # class: 'gd'
Generic.Emph: "italic #000000", # class: 'ge'
Generic.Error: "#ef2929", # class: 'gr'
Generic.Heading: "bold #000080", # class: 'gh'
Generic.Inserted: "#00A000", # class: 'gi'
Generic.Output: "#888", # class: 'go'
Generic.Prompt: "#745334", # class: 'gp'
Generic.Strong: "bold #000000", # class: 'gs'
Generic.Subheading: "bold #800080", # class: 'gu'
Generic.Traceback: "bold #a40000", # class: 'gt'
Name: "#000000", # class: 'n'
Name.Attribute: "#c4a000", # class: 'na' - to be revised
Name.Builtin: "#004461", # class: 'nb'
Name.Builtin.Pseudo: "#3465a4", # class: 'bp'
Name.Class: "#000000", # class: 'nc' - to be revised
Name.Constant: "#000000", # class: 'no' - to be revised
Name.Decorator: "#888", # class: 'nd' - to be revised
Name.Entity: "#ce5c00", # class: 'ni'
Name.Exception: "bold #cc0000", # class: 'ne'
Name.Function: "#000000", # class: 'nf'
Name.Property: "#000000", # class: 'py'
Name.Label: "#f57900", # class: 'nl'
Name.Namespace: "#000000", # class: 'nn' - to be revised
Name.Other: "#000000", # class: 'nx'
Name.Tag: "bold #004461", # class: 'nt' - like a keyword
Name.Variable: "#000000", # class: 'nv' - to be revised
Name.Variable.Class: "#000000", # class: 'vc' - to be revised
Name.Variable.Global: "#000000", # class: 'vg' - to be revised
Name.Variable.Instance: "#000000", # class: 'vi' - to be revised
Number: "#990000", # class: 'm'
Literal: "#000000", # class: 'l'
Literal.Date: "#000000", # class: 'ld'
String: "#4e9a06", # class: 's'
String.Backtick: "#4e9a06", # class: 'sb'
String.Char: "#4e9a06", # class: 'sc'
String.Doc: "italic #8f5902", # class: 'sd' - like a comment
String.Double: "#4e9a06", # class: 's2'
String.Escape: "#4e9a06", # class: 'se'
String.Heredoc: "#4e9a06", # class: 'sh'
String.Interpol: "#4e9a06", # class: 'si'
String.Other: "#4e9a06", # class: 'sx'
String.Regex: "#4e9a06", # class: 'sr'
String.Single: "#4e9a06", # class: 's1'
String.Symbol: "#4e9a06", # class: 'ss'
Generic: "#000000", # class: 'g'
Generic.Deleted: "#a40000", # class: 'gd'
Generic.Emph: "italic #000000", # class: 'ge'
Generic.Error: "#ef2929", # class: 'gr'
Generic.Heading: "bold #000080", # class: 'gh'
Generic.Inserted: "#00A000", # class: 'gi'
Generic.Output: "#888", # class: 'go'
Generic.Prompt: "#745334", # class: 'gp'
Generic.Strong: "bold #000000", # class: 'gs'
Generic.Subheading: "bold #800080", # class: 'gu'
Generic.Traceback: "bold #a40000", # class: 'gt'
}
requests/__init__.py
+22 -17
View File
@@ -1,10 +1,8 @@
# -*- coding: utf-8 -*-
# __
# /__) _ _ _ _ _/ _
# / ( (- (/ (/ (- _) / _)
# /
"""
Requests HTTP Library
~~~~~~~~~~~~~~~~~~~~~
@@ -49,11 +47,9 @@ from .exceptions import RequestsDependencyWarning
def check_compatibility(urllib3_version, chardet_version):
urllib3_version = urllib3_version.split('.')
assert urllib3_version != ['dev'] # Verify urllib3 isn't installed from git.
# Sometimes, urllib3 only reports its version as 16.1.
if len(urllib3_version) == 2:
urllib3_version.append('0')
# Check urllib3 for compatibility.
major, minor, patch = urllib3_version # noqa: F811
major, minor, patch = int(major), int(minor), int(patch)
@@ -61,7 +57,6 @@ def check_compatibility(urllib3_version, chardet_version):
assert major == 1
assert minor >= 21
assert minor <= 22
# Check chardet for compatibility.
major, minor, patch = chardet_version.split('.')[:3]
major, minor, patch = int(major), int(minor), int(patch)
@@ -79,45 +74,56 @@ def _check_cryptography(cryptography_version):
return
if cryptography_version < [1, 3, 4]:
warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version)
warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(
cryptography_version
)
warnings.warn(warning, RequestsDependencyWarning)
# Check imported dependencies for compatibility.
try:
check_compatibility(urllib3.__version__, chardet.__version__)
except (AssertionError, ValueError):
warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported "
"version!".format(urllib3.__version__, chardet.__version__),
RequestsDependencyWarning)
warnings.warn(
"urllib3 ({0}) or chardet ({1}) doesn't match a supported "
"version!".format(urllib3.__version__, chardet.__version__),
RequestsDependencyWarning,
)
# Attempt to enable urllib3's SNI support, if possible
try:
from urllib3.contrib import pyopenssl
pyopenssl.inject_into_urllib3()
pyopenssl.inject_into_urllib3()
# Check cryptography version
from cryptography import __version__ as cryptography_version
_check_cryptography(cryptography_version)
except ImportError:
pass
# urllib3's DependencyWarnings should be silenced.
from urllib3.exceptions import DependencyWarning
warnings.simplefilter('ignore', DependencyWarning)
from .__version__ import __title__, __description__, __url__, __version__
from .__version__ import __build__, __author__, __author_email__, __license__
from .__version__ import __copyright__, __cake__
from . import utils
from .import utils
from .models import Request, Response, PreparedRequest
from .api import request, get, head, post, patch, put, delete, options
from .sessions import session, Session
from .status_codes import codes
from .exceptions import (
RequestException, Timeout, URLRequired,
TooManyRedirects, HTTPError, ConnectionError,
FileModeWarning, ConnectTimeout, ReadTimeout
RequestException,
Timeout,
URLRequired,
TooManyRedirects,
HTTPError,
ConnectionError,
FileModeWarning,
ConnectTimeout,
ReadTimeout,
)
# Set default logging handler to avoid "No handler found" warnings.
@@ -125,6 +131,5 @@ import logging
from logging import NullHandler
logging.getLogger(__name__).addHandler(NullHandler())
# FileModeWarnings go off per the default.
warnings.simplefilter('default', FileModeWarning, append=True)
requests/__version__.py
-1
View File
@@ -1,7 +1,6 @@
# .-. .-. .-. . . .-. .-. .-. .-.
# |( |- |.| | | |- `-. | `-.
# ' ' `-' `-`.`-' `-' `-' ' `-'
__title__ = 'requests'
__description__ = 'Python HTTP for Humans.'
__url__ = 'http://python-requests.org'
requests/_internal_utils.py
+1 -2
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests._internal_utils
~~~~~~~~~~~~~~
@@ -20,7 +19,6 @@ def to_native_string(string, encoding='ascii'):
out = string
else:
out = string.decode(encoding)
return out
@@ -35,5 +33,6 @@ def unicode_is_ascii(u_string):
try:
u_string.encode('ascii')
return True
except UnicodeEncodeError:
return False
requests/adapters.py
+77 -77
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests.adapters
~~~~~~~~~~~~~~~~~
@@ -28,21 +27,35 @@ from urllib3.exceptions import ResponseError
from .models import Response
from .basics import urlparse, basestring
from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
prepend_scheme_if_needed, get_auth_from_url, urldefragauth,
select_proxy)
from .utils import (
DEFAULT_CA_BUNDLE_PATH,
get_encoding_from_headers,
prepend_scheme_if_needed,
get_auth_from_url,
urldefragauth,
select_proxy,
)
from .structures import CaseInsensitiveDict
from .cookies import extract_cookies_to_jar
from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError,
ProxyError, RetryError, InvalidScheme)
from .exceptions import (
ConnectionError,
ConnectTimeout,
ReadTimeout,
SSLError,
ProxyError,
RetryError,
InvalidScheme,
)
from .auth import _basic_auth_str
try:
from urllib3.contrib.socks import SOCKSProxyManager
except ImportError:
def SOCKSProxyManager(*args, **kwargs):
raise InvalidScheme("Missing dependencies for SOCKS support.")
DEFAULT_POOLBLOCK = False
DEFAULT_POOLSIZE = 10
DEFAULT_RETRIES = 0
@@ -64,22 +77,19 @@ def _pool_kwargs(verify, cert):
"""
pool_kwargs = {}
if verify:
cert_loc = None
# Allow self-specified cert location.
if verify is not True:
cert_loc = verify
if not cert_loc:
cert_loc = DEFAULT_CA_BUNDLE_PATH
if not cert_loc or not os.path.exists(cert_loc):
raise IOError("Could not find a suitable TLS CA certificate bundle, "
"invalid path: {0}".format(cert_loc))
raise IOError(
"Could not find a suitable TLS CA certificate bundle, "
"invalid path: {0}".format(cert_loc)
)
pool_kwargs['cert_reqs'] = 'CERT_REQUIRED'
if not os.path.isdir(cert_loc):
pool_kwargs['ca_certs'] = cert_loc
pool_kwargs['ca_cert_dir'] = None
@@ -90,7 +100,6 @@ def _pool_kwargs(verify, cert):
pool_kwargs['cert_reqs'] = 'CERT_NONE'
pool_kwargs['ca_certs'] = None
pool_kwargs['ca_cert_dir'] = None
if cert:
if not isinstance(cert, basestring):
pool_kwargs['cert_file'] = cert[0]
@@ -98,15 +107,19 @@ def _pool_kwargs(verify, cert):
else:
pool_kwargs['cert_file'] = cert
pool_kwargs['key_file'] = None
cert_file = pool_kwargs['cert_file']
key_file = pool_kwargs['key_file']
if cert_file and not os.path.exists(cert_file):
raise IOError("Could not find the TLS certificate file, "
"invalid path: {0}".format(cert_file))
raise IOError(
"Could not find the TLS certificate file, "
"invalid path: {0}".format(cert_file)
)
if key_file and not os.path.exists(key_file):
raise IOError("Could not find the TLS key file, "
"invalid path: {0}".format(key_file))
raise IOError(
"Could not find the TLS key file, " "invalid path: {0}".format(key_file)
)
return pool_kwargs
@@ -116,8 +129,9 @@ class BaseAdapter(object):
def __init__(self):
super(BaseAdapter, self).__init__()
def send(self, request, stream=False, timeout=None, verify=True,
cert=None, proxies=None):
def send(
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
):
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
@@ -165,25 +179,27 @@ class HTTPAdapter(BaseAdapter):
>>> a = requests.adapters.HTTPAdapter(max_retries=3)
>>> s.mount('http://', a)
"""
__attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize',
'_pool_block']
__attrs__ = [
'max_retries', 'config', '_pool_connections', '_pool_maxsize', '_pool_block'
]
def __init__(self, pool_connections=DEFAULT_POOLSIZE,
pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES,
pool_block=DEFAULT_POOLBLOCK):
def __init__(
self,
pool_connections=DEFAULT_POOLSIZE,
pool_maxsize=DEFAULT_POOLSIZE,
max_retries=DEFAULT_RETRIES,
pool_block=DEFAULT_POOLBLOCK,
):
if max_retries == DEFAULT_RETRIES:
self.max_retries = Retry(0, read=False)
else:
self.max_retries = Retry.from_int(max_retries)
self.config = {}
self.proxy_manager = {}
super(HTTPAdapter, self).__init__()
self._pool_connections = pool_connections
self._pool_maxsize = pool_maxsize
self._pool_block = pool_block
self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
def __getstate__(self):
@@ -194,14 +210,15 @@ class HTTPAdapter(BaseAdapter):
# self.poolmanager uses a lambda function, which isn't pickleable.
self.proxy_manager = {}
self.config = {}
for attr, value in state.items():
setattr(self, attr, value)
self.init_poolmanager(
self._pool_connections, self._pool_maxsize, block=self._pool_block
)
self.init_poolmanager(self._pool_connections, self._pool_maxsize,
block=self._pool_block)
def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs):
def init_poolmanager(
self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
):
"""Initializes a urllib3 PoolManager.
This method should not be called from user code, and is only
@@ -217,9 +234,13 @@ class HTTPAdapter(BaseAdapter):
self._pool_connections = connections
self._pool_maxsize = maxsize
self._pool_block = block
self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize,
block=block, strict=True, **pool_kwargs)
self.poolmanager = PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
strict=True,
**pool_kwargs,
)
def proxy_manager_for(self, proxy, **proxy_kwargs):
"""Return urllib3 ProxyManager for the given proxy.
@@ -244,7 +265,7 @@ class HTTPAdapter(BaseAdapter):
num_pools=self._pool_connections,
maxsize=self._pool_maxsize,
block=self._pool_block,
**proxy_kwargs
**proxy_kwargs,
)
else:
proxy_headers = self.proxy_headers(proxy)
@@ -254,8 +275,8 @@ class HTTPAdapter(BaseAdapter):
num_pools=self._pool_connections,
maxsize=self._pool_maxsize,
block=self._pool_block,
**proxy_kwargs)
**proxy_kwargs,
)
return manager
def build_response(self, req, resp):
@@ -269,30 +290,23 @@ class HTTPAdapter(BaseAdapter):
:rtype: requests.Response
"""
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
# Add new cookies from the server.
extract_cookies_to_jar(response.cookies, req, resp)
# Give the Response some context.
response.request = req
response.connection = self
return response
def get_connection(self, url, proxies=None, verify=None, cert=None):
@@ -306,7 +320,6 @@ class HTTPAdapter(BaseAdapter):
"""
pool_kwargs = _pool_kwargs(verify, cert)
proxy = select_proxy(url, proxies)
if proxy:
proxy = prepend_scheme_if_needed(proxy, 'http')
proxy_manager = self.proxy_manager_for(proxy)
@@ -316,7 +329,6 @@ class HTTPAdapter(BaseAdapter):
parsed = urlparse(url)
url = parsed.geturl()
conn = self.poolmanager.connection_from_url(url, pool_kwargs=pool_kwargs)
return conn
def close(self):
@@ -345,17 +357,14 @@ class HTTPAdapter(BaseAdapter):
"""
proxy = select_proxy(request.url, proxies)
scheme = urlparse(request.url).scheme
is_proxied_http_request = (proxy and scheme != 'https')
using_socks_proxy = False
if proxy:
proxy_scheme = urlparse(proxy).scheme.lower()
using_socks_proxy = proxy_scheme.startswith('socks')
url = request.path_url
if is_proxied_http_request and not using_socks_proxy:
url = urldefragauth(request.url)
return url
def add_headers(self, request, **kwargs):
@@ -387,14 +396,13 @@ class HTTPAdapter(BaseAdapter):
"""
headers = {}
username, password = get_auth_from_url(proxy)
if username:
headers['Proxy-Authorization'] = _basic_auth_str(username,
password)
headers['Proxy-Authorization'] = _basic_auth_str(username, password)
return headers
def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
def send(
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
):
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
@@ -411,27 +419,26 @@ class HTTPAdapter(BaseAdapter):
:rtype: requests.Response
"""
conn = self.get_connection(request.url, proxies, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(request)
chunked = not (request.body is None or 'Content-Length' in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
timeout = TimeoutSauce(connect=connect, read=read)
except ValueError as e:
# this may raise a string formatting error.
err = ("Invalid timeout {0}. Pass a (connect, read) "
"timeout tuple, or a single float to set "
"both timeouts to the same value".format(timeout))
err = (
"Invalid timeout {0}. Pass a (connect, read) "
"timeout tuple, or a single float to set "
"both timeouts to the same value".format(timeout)
)
raise ValueError(err)
elif isinstance(timeout, TimeoutSauce):
pass
else:
timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
if not chunked:
resp = conn.urlopen(
@@ -445,36 +452,28 @@ class HTTPAdapter(BaseAdapter):
decode_content=False,
retries=self.max_retries,
timeout=timeout,
enforce_content_length=True
enforce_content_length=True,
)
# Send the request.
else:
if hasattr(conn, 'proxy_pool'):
conn = conn.proxy_pool
low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)
try:
low_conn.putrequest(request.method,
url,
skip_accept_encoding=True)
low_conn.putrequest(request.method, url, skip_accept_encoding=True)
for header, value in request.headers.items():
low_conn.putheader(header, value)
low_conn.endheaders()
for i in request.body:
chunk_size = len(i)
if chunk_size == 0:
continue
low_conn.send(hex(chunk_size)[2:].encode('utf-8'))
low_conn.send(b'\r\n')
low_conn.send(i)
low_conn.send(b'\r\n')
low_conn.send(b'0\r\n\r\n')
# Receive the response from the server
try:
# For Python 2.7, use buffering of HTTP responses
@@ -482,7 +481,6 @@ class HTTPAdapter(BaseAdapter):
except TypeError:
# For Python 3.3+ versions, this is the default
r = low_conn.getresponse()
resp = HTTPResponse.from_httplib(
r,
pool=conn,
@@ -490,7 +488,7 @@ class HTTPAdapter(BaseAdapter):
preload_content=False,
decode_content=False,
enforce_content_length=True,
request_method=request.method
request_method=request.method,
)
except:
# If we hit any problems here, clean up the connection.
@@ -529,8 +527,10 @@ class HTTPAdapter(BaseAdapter):
if isinstance(e, _SSLError):
# This branch is for urllib3 versions earlier than v1.22
raise SSLError(e, request=request)
elif isinstance(e, ReadTimeoutError):
raise ReadTimeout(e, request=request)
else:
raise
requests/api.py
+8 -15
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests.api
~~~~~~~~~~~~
@@ -10,11 +9,13 @@ This module implements the Requests API.
:license: Apache2, see LICENSE for more details.
"""
from . import sessions
from . import types
from .import sessions
from .import types
def request(method: types.Method, url: types.URL, *, session: types.Session = None, **kwargs) -> types.Response:
def request(
method: types.Method, url: types.URL, *, session: types.Session = None, **kwargs
) -> types.Response:
"""Constructs and sends a :class:`Request <Request>`.
:param method: method for the new :class:`Request` object.
@@ -52,13 +53,10 @@ def request(method: types.Method, url: types.URL, *, session: types.Session = No
>>> req = requests.request('GET', 'http://httpbin.org/get')
<Response [200]>
"""
# By using the 'with' statement we are sure the session is closed, thus we
# avoid leaving sockets open which can trigger a ResourceWarning in some
# cases, and look like a memory leak in others.
session = sessions.Session() if session is None else session
with session:
return session.request(method=method, url=url, **kwargs)
@@ -72,7 +70,6 @@ def get(url: types.URL, *, params: types.Params = None, **kwargs) -> types.Respo
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
kwargs.setdefault('allow_redirects', True)
return request('get', url, params=params, **kwargs)
@@ -85,7 +82,6 @@ def options(url: types.URL, **kwargs) -> types.Response:
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
kwargs.setdefault('allow_redirects', True)
return request('options', url, **kwargs)
@@ -98,12 +94,13 @@ def head(url: types.URL, **kwargs) -> types.Response:
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
kwargs.setdefault('allow_redirects', False)
return request('head', url, **kwargs)
def post(url: types.URL, *, data: types.Data = None, json: types.JSON = None, **kwargs) -> types.Response:
def post(
url: types.URL, *, data: types.Data = None, json: types.JSON = None, **kwargs
) -> types.Response:
r"""Sends a POST request.
:param url: URL for the new :class:`Request` object.
@@ -113,7 +110,6 @@ def post(url: types.URL, *, data: types.Data = None, json: types.JSON = None, **
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request('post', url, data=data, json=json, **kwargs)
@@ -127,7 +123,6 @@ def put(url: types.URL, *, data: types.Data = None, **kwargs) -> types.Response:
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request('put', url, data=data, **kwargs)
@@ -141,7 +136,6 @@ def patch(url: types.URL, *, data: types.Data = None, **kwargs) -> types.Respons
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request('patch', url, data=data, **kwargs)
@@ -153,5 +147,4 @@ def delete(url: types.URL, **kwargs) -> types.Response:
:return: :class:`Response <Response>` object
:rtype: requests.Response
"""
return request('delete', url, **kwargs)
requests/auth.py
+31 -42
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests.auth
~~~~~~~~~~~~~
@@ -26,25 +25,25 @@ CONTENT_TYPE_MULTI_PART = 'multipart/form-data'
def _basic_auth_str(username, password):
"""Returns a Basic Auth string."""
if not isinstance(username, basestring):
raise TypeError('username must be of type str or bytes, '
'instead it was %s' % type(username))
raise TypeError(
'username must be of type str or bytes, '
'instead it was %s' % type(username)
)
if not isinstance(password, basestring):
raise TypeError('password must be of type str or bytes, '
'instead it was %s' % type(password))
raise TypeError(
'password must be of type str or bytes, '
'instead it was %s' % type(password)
)
if isinstance(username, str):
username = username.encode('latin1')
if isinstance(password, str):
password = password.encode('latin1')
authstr = 'Basic ' + to_native_string(
b64encode(b':'.join((username, password))).strip()
)
return authstr
@@ -63,10 +62,12 @@ class HTTPBasicAuth(AuthBase):
self.password = password
def __eq__(self, other):
return all([
self.username == getattr(other, 'username', None),
self.password == getattr(other, 'password', None)
])
return all(
[
self.username == getattr(other, 'username', None),
self.password == getattr(other, 'password', None),
]
)
def __ne__(self, other):
return not self == other
@@ -99,51 +100,48 @@ class HTTPDigestAuth(AuthBase):
"""
:rtype: str
"""
realm = self._thread_local.chal['realm']
nonce = self._thread_local.chal['nonce']
qop = self._thread_local.chal.get('qop')
algorithm = self._thread_local.chal.get('algorithm')
opaque = self._thread_local.chal.get('opaque')
hash_utf8 = None
if algorithm is None:
_algorithm = 'MD5'
else:
_algorithm = algorithm.upper()
# lambdas assume digest modules are imported at the top level
if _algorithm == 'MD5' or _algorithm == 'MD5-SESS':
def md5_utf8(x):
if isinstance(x, str):
x = x.encode('utf-8')
return hashlib.md5(x).hexdigest()
hash_utf8 = md5_utf8
elif _algorithm == 'SHA':
def sha_utf8(x):
if isinstance(x, str):
x = x.encode('utf-8')
return hashlib.sha1(x).hexdigest()
hash_utf8 = sha_utf8
KD = lambda s, d: hash_utf8("%s:%s" % (s, d))
if hash_utf8 is None:
return None
# XXX not implemented yet
entdig = None
p_parsed = urlparse(url)
#: path is request-uri defined in RFC 2616 which should not be empty
# : path is request-uri defined in RFC 2616 which should not be empty
path = p_parsed.path or "/"
if p_parsed.query:
path += '?' + p_parsed.query
A1 = '%s:%s:%s' % (self.username, realm, self.password)
A2 = '%s:%s' % (method, path)
HA1 = hash_utf8(A1)
HA2 = hash_utf8(A2)
if nonce == self._thread_local.last_nonce:
self._thread_local.nonce_count += 1
else:
@@ -153,27 +151,23 @@ class HTTPDigestAuth(AuthBase):
s += nonce.encode('utf-8')
s += time.ctime().encode('utf-8')
s += os.urandom(8)
cnonce = (hashlib.sha1(s).hexdigest()[:16])
if _algorithm == 'MD5-SESS':
HA1 = hash_utf8('%s:%s:%s' % (HA1, nonce, cnonce))
if not qop:
respdig = KD(HA1, "%s:%s" % (nonce, HA2))
elif qop == 'auth' or 'auth' in qop.split(','):
noncebit = "%s:%s:%s:%s:%s" % (
nonce, ncvalue, cnonce, 'auth', HA2
)
noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, 'auth', HA2)
respdig = KD(HA1, noncebit)
else:
# XXX handle auth-int.
return None
self._thread_local.last_nonce = nonce
# XXX should the partial digests be encoded too?
base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
'response="%s"' % (self.username, realm, nonce, path, respdig)
base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' 'response="%s"' % (
self.username, realm, nonce, path, respdig
)
if opaque:
base += ', opaque="%s"' % opaque
if algorithm:
@@ -182,7 +176,6 @@ class HTTPDigestAuth(AuthBase):
base += ', digest="%s"' % entdig
if qop:
base += ', qop="auth", nc=%s, cnonce="%s"' % (ncvalue, cnonce)
return 'Digest %s' % (base)
def handle_redirect(self, r, **kwargs):
@@ -196,7 +189,6 @@ class HTTPDigestAuth(AuthBase):
:rtype: requests.Response
"""
# If response is not 4xx, do not auth
# See https://github.com/requests/requests/issues/3772
if not 400 <= r.status_code < 500:
@@ -208,13 +200,10 @@ class HTTPDigestAuth(AuthBase):
# it was to resend the request.
r.request.body.seek(self._thread_local.pos)
s_auth = r.headers.get('www-authenticate', '')
if 'digest' in s_auth.lower() and self._thread_local.num_401_calls < 2:
self._thread_local.num_401_calls += 1
pat = re.compile(r'digest ', flags=re.IGNORECASE)
self._thread_local.chal = parse_dict_header(pat.sub('', s_auth, count=1))
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
@@ -222,13 +211,12 @@ class HTTPDigestAuth(AuthBase):
prep = r.request.copy()
extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Authorization'] = self.build_digest_header(
prep.method, prep.url)
prep.method, prep.url
)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
self._thread_local.num_401_calls = 1
@@ -251,14 +239,15 @@ class HTTPDigestAuth(AuthBase):
r.register_hook('response', self.handle_401)
r.register_hook('response', self.handle_redirect)
self._thread_local.num_401_calls = 1
return r
def __eq__(self, other):
return all([
self.username == getattr(other, 'username', None),
self.password == getattr(other, 'password', None)
])
return all(
[
self.username == getattr(other, 'username', None),
self.password == getattr(other, 'password', None),
]
)
def __ne__(self, other):
return not self == other
requests/basics.py
+18 -10
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests.basics
~~~~~~~~~~~~~~~
@@ -14,23 +13,32 @@ import sys
# ---------
# Specifics
# ---------
from urllib.parse import (
urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote,
quote_plus, unquote_plus, urldefrag
urlparse,
urlunparse,
urljoin,
urlsplit,
urlencode,
quote,
unquote,
quote_plus,
unquote_plus,
urldefrag,
)
from urllib.request import (
parse_http_list, getproxies,
proxy_bypass, proxy_bypass_environment, getproxies_environment
parse_http_list,
getproxies,
proxy_bypass,
proxy_bypass_environment,
getproxies_environment,
)
from http import cookiejar as cookielib
from http.cookies import Morsel
from io import StringIO
builtin_str = str # type: ignore
str = str # type: ignore
bytes = bytes # type: ignore
builtin_str = str # type: ignore
str = str # type: ignore
bytes = bytes # type: ignore
basestring = (str, bytes)
numeric_types = (int, float)
integer_types = (int,)
requests/certs.py
-1
View File
@@ -1,6 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
requests.certs
~~~~~~~~~~~~~~
requests/cookies.py
+40 -25
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests.cookies
~~~~~~~~~~~~~~~~
@@ -20,7 +19,7 @@ from .basics import cookielib, urlparse, urlunparse, Morsel
try:
import threading
except ImportError:
import dummy_threading as threading # type: ignore
import dummy_threading as threading # type: ignore
class MockRequest(object):
@@ -54,14 +53,21 @@ class MockRequest(object):
# header
if not self._r.headers.get('Host'):
return self._r.url
# If they did set it, retrieve it and reconstruct the expected domain
host = to_native_string(self._r.headers['Host'], encoding='utf-8')
parsed = urlparse(self._r.url)
# Reconstruct the URL as we expect it
return urlunparse([
parsed.scheme, host, parsed.path, parsed.params, parsed.query,
parsed.fragment
])
return urlunparse(
[
parsed.scheme,
host,
parsed.path,
parsed.params,
parsed.query,
parsed.fragment,
]
)
def is_unverifiable(self):
return True
@@ -74,7 +80,9 @@ class MockRequest(object):
def add_header(self, key, val):
"""cookielib has no legitimate use for this method; add it back if you find one."""
raise NotImplementedError("Cookie headers should be added with add_unredirected_header()")
raise NotImplementedError(
"Cookie headers should be added with add_unredirected_header()"
)
def add_unredirected_header(self, name, value):
self._new_headers[name] = value
@@ -123,9 +131,9 @@ def extract_cookies_to_jar(jar, request, response):
:param request: our own requests.Request object
:param response: urllib3.HTTPResponse object
"""
if not (hasattr(response, '_original_response') and
response._original_response):
if not (hasattr(response, '_original_response') and response._original_response):
return
# the _original_response field is the wrapped httplib.HTTPResponse object,
req = MockRequest(request)
# pull out the HTTPMessage with the headers and put it in the mock:
@@ -153,12 +161,14 @@ def remove_cookie_by_name(cookiejar, name, domain=None, path=None):
for cookie in cookiejar:
if cookie.name != name:
continue
if domain is not None and domain != cookie.domain:
continue
if path is not None and path != cookie.path:
continue
clearables.append((cookie.domain, cookie.path, cookie.name))
clearables.append((cookie.domain, cookie.path, cookie.name))
for domain, path, name in clearables:
cookiejar.clear(domain, path, name)
@@ -196,6 +206,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
"""
try:
return self._find_no_duplicates(name, domain, path)
except KeyError:
return default
@@ -206,7 +217,9 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
"""
# support client code that unsets cookies by assignment of a None value:
if value is None:
remove_cookie_by_name(self, name, domain=kwargs.get('domain'), path=kwargs.get('path'))
remove_cookie_by_name(
self, name, domain=kwargs.get('domain'), path=kwargs.get('path')
)
return
if isinstance(value, Morsel):
@@ -294,6 +307,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
for cookie in iter(self):
if cookie.domain is not None and cookie.domain in domains:
return True
domains.append(cookie.domain)
return False # there is only one domain in jar
@@ -316,6 +330,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
def __contains__(self, name):
try:
return super(RequestsCookieJar, self).__contains__(name)
except CookieConflictError:
return True
@@ -342,7 +357,11 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
remove_cookie_by_name(self, name)
def set_cookie(self, cookie, *args, **kwargs):
if hasattr(cookie.value, 'startswith') and cookie.value.startswith('"') and cookie.value.endswith('"'):
if hasattr(cookie.value, 'startswith') and cookie.value.startswith(
'"'
) and cookie.value.endswith(
'"'
):
cookie.value = cookie.value.replace('\\"', '')
return super(RequestsCookieJar, self).set_cookie(cookie, *args, **kwargs)
@@ -392,11 +411,14 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
if domain is None or cookie.domain == domain:
if path is None or cookie.path == path:
if toReturn is not None: # if there are multiple cookies that meet passed in criteria
raise CookieConflictError('There are multiple cookies with name, %r' % (name))
toReturn = cookie.value # we will eventually return this as long as no cookie conflict
raise CookieConflictError(
'There are multiple cookies with name, %r' % (name)
)
toReturn = cookie.value # we will eventually return this as long as no cookie conflict
if toReturn:
return toReturn
raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path))
def __getstate__(self):
@@ -426,6 +448,7 @@ def _copy_cookie_jar(jar):
if hasattr(jar, 'copy'):
# We're dealing with an instance of RequestsCookieJar
return jar.copy()
# We're dealing with a generic CookieJar instance
new_jar = copy.copy(jar)
new_jar.clear()
@@ -455,7 +478,6 @@ def create_cookie(name, value, **kwargs):
'rest': {'HttpOnly': None},
'rfc2109': False,
}
badargs = set(kwargs) - set(result)
if badargs:
err = 'create_cookie() got unexpected keyword arguments: %s'
@@ -466,24 +488,21 @@ def create_cookie(name, value, **kwargs):
result['domain_specified'] = bool(result['domain'])
result['domain_initial_dot'] = result['domain'].startswith('.')
result['path_specified'] = bool(result['path'])
return cookielib.Cookie(**result)
def morsel_to_cookie(morsel):
"""Convert a Morsel object into a Cookie containing the one k/v pair."""
expires = None
if morsel['max-age']:
try:
expires = int(time.time() + int(morsel['max-age']))
except ValueError:
raise TypeError('max-age: %s must be integer' % morsel['max-age'])
elif morsel['expires']:
time_template = '%a, %d-%b-%Y %H:%M:%S GMT'
expires = calendar.timegm(
time.strptime(morsel['expires'], time_template)
)
expires = calendar.timegm(time.strptime(morsel['expires'], time_template))
return create_cookie(
comment=morsel['comment'],
comment_url=bool(morsel['comment']),
@@ -511,13 +530,11 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True):
"""
if cookiejar is None:
cookiejar = RequestsCookieJar()
if cookie_dict is not None:
names_from_jar = [cookie.name for cookie in cookiejar]
for name in cookie_dict:
if overwrite or (name not in names_from_jar):
cookiejar.set_cookie(create_cookie(name, cookie_dict[name]))
return cookiejar
@@ -531,13 +548,11 @@ def merge_cookies(cookiejar, cookies):
raise ValueError('You can only merge into CookieJar')
if isinstance(cookies, dict):
cookiejar = cookiejar_from_dict(
cookies, cookiejar=cookiejar, overwrite=False)
cookiejar = cookiejar_from_dict(cookies, cookiejar=cookiejar, overwrite=False)
elif isinstance(cookies, cookielib.CookieJar):
try:
cookiejar.update(cookies)
except AttributeError:
for cookie_in_jar in cookies:
cookiejar.set_cookie(cookie_in_jar)
return cookiejar
requests/exceptions.py
+4 -5
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests.exceptions
~~~~~~~~~~~~~~~~~~~
@@ -19,8 +18,7 @@ class RequestException(IOError):
response = kwargs.pop('response', None)
self.response = response
self.request = kwargs.pop('request', None)
if (response is not None and not self.request and
hasattr(response, 'request')):
if (response is not None and not self.request and hasattr(response, 'request')):
self.request = self.response.request
super(RequestException, self).__init__(*args, **kwargs)
@@ -108,9 +106,10 @@ class UnrewindableBodyError(RequestException):
class InvalidBodyError(RequestException, ValueError):
"""An invalid request body was specified"""
# Warnings
class RequestsWarning(Warning):
"""Base warning for Requests."""
pass
requests/help.py
+18 -39
View File
@@ -10,12 +10,12 @@ import idna
import urllib3
import chardet
from . import types
from .import types
from . import __version__ as requests_version
from .import __version__ as requests_version
try:
from .packages.urllib3.contrib import pyopenssl
from . packages.urllib3.contrib import pyopenssl
except ImportError:
pyopenssl = None
OpenSSL = None
@@ -37,66 +37,47 @@ def _implementation() -> types.Help:
to work out the correct shape of the code for those platforms.
"""
implementation = platform.python_implementation()
if implementation == 'CPython':
implementation_version = platform.python_version()
elif implementation == 'PyPy':
implementation_version = '%s.%s.%s' % (sys.pypy_version_info.major,
sys.pypy_version_info.minor,
sys.pypy_version_info.micro)
implementation_version = '%s.%s.%s' % (
sys.pypy_version_info.major,
sys.pypy_version_info.minor,
sys.pypy_version_info.micro,
)
if sys.pypy_version_info.releaselevel != 'final':
implementation_version = ''.join([
implementation_version, sys.pypy_version_info.releaselevel
])
implementation_version = ''.join(
[implementation_version, sys.pypy_version_info.releaselevel]
)
elif implementation == 'Jython':
implementation_version = platform.python_version() # Complete Guess
elif implementation == 'IronPython':
implementation_version = platform.python_version() # Complete Guess
else:
implementation_version = 'Unknown'
return {'name': implementation, 'version': implementation_version}
def info() -> types.Help:
"""Generate information for a bug report."""
try:
platform_info = {
'system': platform.system(),
'release': platform.release(),
}
platform_info = {'system': platform.system(), 'release': platform.release()}
except IOError:
platform_info = {
'system': 'Unknown',
'release': 'Unknown',
}
platform_info = {'system': 'Unknown', 'release': 'Unknown'}
implementation_info = _implementation()
urllib3_info = {'version': urllib3.__version__}
chardet_info = {'version': chardet.__version__}
pyopenssl_info = {
'version': None,
'openssl_version': '',
}
pyopenssl_info = {'version': None, 'openssl_version': ''}
if OpenSSL:
pyopenssl_info = {
'version': OpenSSL.__version__,
'openssl_version': '%x' % OpenSSL.SSL.OPENSSL_VERSION_NUMBER,
}
cryptography_info = {
'version': getattr(cryptography, '__version__', ''),
}
idna_info = {
'version': getattr(idna, '__version__', ''),
}
cryptography_info = {'version': getattr(cryptography, '__version__', '')}
idna_info = {'version': getattr(idna, '__version__', '')}
# OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module.
system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None)
system_ssl_info = {
'version': '%x' % system_ssl if system_ssl is not None else ''
}
system_ssl_info = {'version': '%x' % system_ssl if system_ssl is not None else ''}
return {
'platform': platform_info,
'implementation': implementation_info,
@@ -107,9 +88,7 @@ def info() -> types.Help:
'chardet': chardet_info,
'cryptography': cryptography_info,
'idna': idna_info,
'requests': {
'version': requests_version,
},
'requests': {'version': requests_version},
}
requests/hooks.py
+3 -3
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests.hooks
~~~~~~~~~~~~~~
@@ -17,9 +16,10 @@ HOOKS = ['response']
def default_hooks():
return {event: [] for event in HOOKS}
# TODO: response is the only one
def dispatch_hook(key, hooks, hook_data, **kwargs):
"""Dispatches a hook dictionary on a given piece of data."""
hooks = hooks or {}
requests/models.py
+175 -158
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests.models
~~~~~~~~~~~~~~~
@@ -21,7 +20,8 @@ from urllib3.fields import RequestField
from urllib3.filepost import encode_multipart_formdata
from urllib3.util import parse_url
from urllib3.exceptions import (
DecodeError, ReadTimeoutError, ProtocolError, LocationParseError)
DecodeError, ReadTimeoutError, ProtocolError, LocationParseError
)
from io import UnsupportedOperation
from .hooks import default_hooks
@@ -31,59 +31,74 @@ import requests
from .auth import HTTPBasicAuth
from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar
from .exceptions import (
HTTPError, MissingScheme, InvalidURL, ChunkedEncodingError,
ContentDecodingError, ConnectionError, StreamConsumedError,
InvalidHeader, InvalidBodyError, ReadTimeout
HTTPError,
MissingScheme,
InvalidURL,
ChunkedEncodingError,
ContentDecodingError,
ConnectionError,
StreamConsumedError,
InvalidHeader,
InvalidBodyError,
ReadTimeout,
)
from ._internal_utils import to_native_string, unicode_is_ascii
from .utils import (
guess_filename, get_auth_from_url, requote_uri,
stream_decode_response_unicode, to_key_val_list, parse_header_links,
iter_slices, guess_json_utf, super_len, check_header_validity,
is_stream
guess_filename,
get_auth_from_url,
requote_uri,
stream_decode_response_unicode,
to_key_val_list,
parse_header_links,
iter_slices,
guess_json_utf,
super_len,
check_header_validity,
is_stream,
)
from .basics import (
cookielib, urlunparse, urlsplit, urlencode, str, bytes,
chardet, builtin_str, basestring
cookielib,
urlunparse,
urlsplit,
urlencode,
str,
bytes,
chardet,
builtin_str,
basestring,
)
import json as complexjson
from .status_codes import codes
#: The set of HTTP status codes that indicate an automatically
# : The set of HTTP status codes that indicate an automatically
#: processable redirect.
REDIRECT_STATI = (
codes['moved'], # 301
codes['found'], # 302
codes['other'], # 303
codes['moved'], # 301
codes['found'], # 302
codes['other'], # 303
codes['temporary_redirect'], # 307
codes['permanent_redirect'], # 308
)
DEFAULT_REDIRECT_LIMIT = 30
CONTENT_CHUNK_SIZE = 10 * 1024
ITER_CHUNK_SIZE = 512
class RequestEncodingMixin(object):
@property
def path_url(self):
"""Build the path URL to use."""
url = []
p = urlsplit(self.url)
path = p.path
if not path:
path = '/'
url.append(path)
query = p.query
if query:
url.append('?')
url.append(query)
return ''.join(url)
@staticmethod
@@ -94,11 +109,12 @@ class RequestEncodingMixin(object):
2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
if parameters are supplied as a dict.
"""
if isinstance(data, (str, bytes)):
return data
elif hasattr(data, 'read'):
return data
elif hasattr(data, '__iter__'):
result = []
for k, vs in to_key_val_list(data):
@@ -107,9 +123,13 @@ class RequestEncodingMixin(object):
for v in vs:
if v is not None:
result.append(
(k.encode('utf-8') if isinstance(k, str) else k,
v.encode('utf-8') if isinstance(v, str) else v))
(
k.encode('utf-8') if isinstance(k, str) else k,
v.encode('utf-8') if isinstance(v, str) else v,
)
)
return urlencode(result, doseq=True)
else:
return data
@@ -125,13 +145,13 @@ class RequestEncodingMixin(object):
"""
if (not files):
raise ValueError("Files must be provided.")
elif isinstance(data, basestring):
raise ValueError("Data must not be a string.")
new_fields = []
fields = to_key_val_list(data or {})
files = to_key_val_list(files or {})
for field, val in fields:
if isinstance(val, basestring) or not hasattr(val, '__iter__'):
val = [val]
@@ -140,11 +160,14 @@ class RequestEncodingMixin(object):
# Don't call str() on bytestrings: in Py3 it all goes wrong.
if not isinstance(v, bytes):
v = str(v)
new_fields.append(
(field.decode('utf-8') if isinstance(field, bytes) else field,
v.encode('utf-8') if isinstance(v, str) else v))
(
field.decode('utf-8') if isinstance(
field, bytes
) else field,
v.encode('utf-8') if isinstance(v, str) else v,
)
)
for (k, v) in files:
# support for explicit filename
ft = None
@@ -159,41 +182,41 @@ class RequestEncodingMixin(object):
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, (str, bytes, bytearray)):
fdata = fp
else:
fdata = fp.read()
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
rf.make_multipart(content_type=ft)
new_fields.append(rf)
body, content_type = encode_multipart_formdata(new_fields)
return body, content_type
class RequestHooksMixin(object):
def register_hook(self, event, hook):
"""Properly register a hook."""
if event not in self.hooks:
raise ValueError('Unsupported event specified, with event name "%s"' % (event))
raise ValueError(
'Unsupported event specified, with event name "%s"' % (event)
)
if isinstance(hook, collections.Callable):
self.hooks[event].append(hook)
elif hasattr(hook, '__iter__'):
self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable))
self.hooks[event].extend(
h for h in hook if isinstance(h, collections.Callable)
)
def deregister_hook(self, event, hook):
"""Deregister a previously registered hook.
Returns True if the hook existed, False if not.
"""
try:
self.hooks[event].remove(hook)
return True
except ValueError:
return False
@@ -222,21 +245,28 @@ class Request(RequestHooksMixin):
<PreparedRequest [GET]>
"""
def __init__(self,
method=None, url=None, headers=None, files=None, data=None,
params=None, auth=None, cookies=None, hooks=None, json=None):
def __init__(
self,
method=None,
url=None,
headers=None,
files=None,
data=None,
params=None,
auth=None,
cookies=None,
hooks=None,
json=None,
):
# Default empty dicts for dict params.
data = [] if data is None else data
files = [] if files is None else files
headers = {} if headers is None else headers
params = {} if params is None else params
hooks = {} if hooks is None else hooks
self.hooks = default_hooks()
for (k, v) in list(hooks.items()):
self.register_hook(event=k, hook=v)
self.method = method
self.url = url
self.headers = headers
@@ -287,37 +317,44 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
"""
def __init__(self):
#: HTTP verb to send to the server.
# : HTTP verb to send to the server.
self.method = None
#: HTTP URL to send the request to.
# : HTTP URL to send the request to.
self.url = None
#: dictionary of HTTP headers.
# : dictionary of HTTP headers.
self.headers = None
# The `CookieJar` used to create the Cookie header will be stored here
# after prepare_cookies is called
self._cookies = None
#: request body to send to the server.
# : request body to send to the server.
self.body = None
#: dictionary of callback hooks, for internal usage.
# : dictionary of callback hooks, for internal usage.
self.hooks = default_hooks()
#: integer denoting starting position of a readable file-like body.
# : integer denoting starting position of a readable file-like body.
self._body_position = None
def prepare(self,
method=None, url=None, headers=None, files=None, data=None,
params=None, auth=None, cookies=None, hooks=None, json=None):
def prepare(
self,
method=None,
url=None,
headers=None,
files=None,
data=None,
params=None,
auth=None,
cookies=None,
hooks=None,
json=None,
):
"""Prepares the entire request with the given parameters."""
self.prepare_method(method)
self.prepare_url(url, params)
self.prepare_headers(headers)
self.prepare_cookies(cookies)
self.prepare_body(data, files, json)
self.prepare_auth(auth, url)
# Note that prepare_auth must be last to enable authentication schemes
# such as OAuth to work on a fully prepared request.
# This MUST go after prepare_auth. Authenticators could add a hook
self.prepare_hooks(hooks)
@@ -340,6 +377,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
self.method = method
if self.method is None:
raise ValueError('Request method cannot be "None"')
self.method = to_native_string(self.method.upper())
@staticmethod
@@ -350,11 +388,12 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
host = idna.encode(host, uts46=True).decode('utf-8')
except idna.IDNAError:
raise UnicodeError
return host
def prepare_url(self, url, params):
"""Prepares the given HTTP URL."""
#: Accept objects that have string representations.
# : Accept objects that have string representations.
#: We're unable to blindly call unicode/str functions
#: as this will include the bytestring indicator (b'')
#: on python 3.x.
@@ -363,10 +402,8 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
url = url.decode('utf8')
else:
url = str(url)
# Ignore any leading and trailing whitespace characters.
url = url.strip()
# Don't do any URL preparation for non-HTTP schemes like `mailto`,
# `data` etc to work around exceptions from `url_parse`, which
# handles RFC 3986 only.
@@ -378,12 +415,13 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
try:
scheme, auth, host, port, path, query, fragment = parse_url(url)
except LocationParseError as e:
raise InvalidURL(*e.args)
raise InvalidURL(* e.args)
if not scheme:
error = ("Invalid URL {0!r}: No scheme supplied. Perhaps you meant http://{0}?")
error = (
"Invalid URL {0!r}: No scheme supplied. Perhaps you meant http://{0}?"
)
error = error.format(to_native_string(url, 'utf8'))
raise MissingScheme(error)
if not host:
@@ -398,6 +436,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
host = self._get_idna_encoded_host(host)
except UnicodeError:
raise InvalidURL('URL has an invalid label.')
elif host.startswith(u'*'):
raise InvalidURL('URL has an invalid label.')
@@ -408,27 +447,22 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
netloc += host
if port:
netloc += ':' + str(port)
# Bare domains aren't valid URLs.
if not path:
path = '/'
if isinstance(params, (str, bytes)):
params = to_native_string(params)
enc_params = self._encode_params(params)
if enc_params:
if query:
query = '%s&%s' % (query, enc_params)
else:
query = enc_params
url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment]))
self.url = url
def prepare_headers(self, headers):
"""Prepares the given HTTP headers."""
self.headers = CaseInsensitiveDict()
if headers:
for header in headers.items():
@@ -439,14 +473,11 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
def prepare_body(self, data, files, json=None):
"""Prepares the given HTTP body data."""
# Check if file, fo, generator, iterator.
# If not, run through normal process.
# Nottin' on you.
body = None
content_type = None
if not data and json is not None:
# urllib3 requires a bytes-like body. Python 2's json.dumps
# provides this natively, but Python 3 gives a Unicode string.
@@ -454,10 +485,8 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
body = complexjson.dumps(json)
if not isinstance(body, bytes):
body = body.encode('utf-8')
if is_stream(data):
body = data
if getattr(body, 'tell', None) is not None:
# Record the current file position before reading.
# This will allow us to rewind a file in the event
@@ -468,9 +497,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
# This differentiates from None, allowing us to catch
# a failed `tell()` later when trying to rewind the body
self._body_position = object()
if files:
raise NotImplementedError('Streamed bodies and files are mutually exclusive.')
raise NotImplementedError(
'Streamed bodies and files are mutually exclusive.'
)
else:
# Multi-part file uploads.
@@ -483,11 +513,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
content_type = None
else:
content_type = 'application/x-www-form-urlencoded'
# Add content-type if it wasn't explicitly provided.
if content_type and ('content-type' not in self.headers):
self.headers['Content-Type'] = content_type
self.prepare_content_length(body)
self.body = body
@@ -502,41 +530,41 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
"""
if body is not None:
length = super_len(body)
if length:
self.headers['Content-Length'] = builtin_str(length)
elif is_stream(body):
self.headers['Transfer-Encoding'] = 'chunked'
else:
raise InvalidBodyError('Non-null body must have length or be streamable.')
elif self.method not in ('GET', 'HEAD') and self.headers.get('Content-Length') is None:
raise InvalidBodyError(
'Non-null body must have length or be streamable.'
)
elif self.method not in ('GET', 'HEAD') and self.headers.get(
'Content-Length'
) is None:
# Set Content-Length to 0 for methods that can have a body
# but don't provide one. (i.e. not GET or HEAD)
self.headers['Content-Length'] = '0'
if 'Transfer-Encoding' in self.headers and 'Content-Length' in self.headers:
raise InvalidHeader('Conflicting Headers: Both Transfer-Encoding and '
'Content-Length are set.')
raise InvalidHeader(
'Conflicting Headers: Both Transfer-Encoding and '
'Content-Length are set.'
)
def prepare_auth(self, auth, url=''):
"""Prepares the given HTTP auth data."""
# If no Auth is explicitly provided, extract it from the URL first.
if auth is None:
url_auth = get_auth_from_url(self.url)
auth = url_auth if any(url_auth) else None
if auth:
if isinstance(auth, tuple) and len(auth) == 2:
# special-case basic HTTP auth
auth = HTTPBasicAuth(*auth)
# Allow auth to make its changes.
r = auth(self)
# Update self to reflect the auth changes.
self.__dict__.update(r.__dict__)
# Recompute Content-Length
self.prepare_content_length(self.body)
@@ -555,7 +583,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
self._cookies = cookies
else:
self._cookies = cookiejar_from_dict(cookies)
cookie_header = get_cookie_header(self._cookies, self)
if cookie_header is not None:
self.headers['Cookie'] = cookie_header
@@ -573,7 +600,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
"""Sends the PreparedRequest to the given Session.
If none is provided, one is created for you."""
session = requests.Session() if session is None else session
with session:
return session.send(self, **send_kwargs)
@@ -582,57 +608,54 @@ class Response(object):
"""The :class:`Response <Response>` object, which contains a
server's response to an HTTP request.
"""
__attrs__ = [
'_content', 'status_code', 'headers', 'url', 'history',
'encoding', 'reason', 'cookies', 'elapsed', 'request'
'_content',
'status_code',
'headers',
'url',
'history',
'encoding',
'reason',
'cookies',
'elapsed',
'request',
]
def __init__(self):
self._content = False
self._content_consumed = False
self._next = None
#: Integer Code of responded HTTP Status, e.g. 404 or 200.
# : Integer Code of responded HTTP Status, e.g. 404 or 200.
self.status_code = None
#: Case-insensitive Dictionary of Response Headers.
# : Case-insensitive Dictionary of Response Headers.
#: For example, ``headers['content-encoding']`` will return the
#: value of a ``'Content-Encoding'`` response header.
self.headers = CaseInsensitiveDict()
#: File-like object representation of response (for advanced usage).
# : File-like object representation of response (for advanced usage).
#: Use of ``raw`` requires that ``stream=True`` be set on the request.
# This requirement does not apply for use internally to Requests.
self.raw = None
#: Final URL location of Response.
# : Final URL location of Response.
self.url = None
#: Encoding to decode with when accessing r.text or
# : Encoding to decode with when accessing r.text or
#: r.iter_content(decode_unicode=True)
self.encoding = None
#: A list of :class:`Response <Response>` objects from
# : A list of :class:`Response <Response>` objects from
#: the history of the Request. Any redirect responses will end
#: up here. The list is sorted from the oldest to the most recent request.
self.history = []
#: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".
# : Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".
self.reason = None
#: A CookieJar of Cookies the server sent back.
# : A CookieJar of Cookies the server sent back.
self.cookies = cookiejar_from_dict({})
#: The amount of time elapsed between sending the request
# : The amount of time elapsed between sending the request
#: and the arrival of the response (as a timedelta).
#: This property specifically measures the time taken between sending
#: the first byte of the request and finishing parsing the headers. It
#: is therefore unaffected by consuming the response content or the
#: value of the ``stream`` keyword argument.
self.elapsed = datetime.timedelta(0)
#: The :class:`PreparedRequest <PreparedRequest>` object to which this
# : The :class:`PreparedRequest <PreparedRequest>` object to which this
#: is a response.
self.request = None
@@ -647,13 +670,11 @@ class Response(object):
# sure the content has been fully read.
if not self._content_consumed:
self.content
return {attr: getattr(self, attr, None) for attr in self.__attrs__}
def __setstate__(self, state):
for name, value in state.items():
setattr(self, name, value)
# pickled objects do not have .raw
setattr(self, '_content_consumed', True)
setattr(self, 'raw', None)
@@ -678,6 +699,7 @@ class Response(object):
self.raise_for_status()
except HTTPError:
return False
return True
@property
@@ -690,7 +712,10 @@ class Response(object):
@property
def is_permanent_redirect(self):
"""True if this Response one of the permanent versions of redirect."""
return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect))
return (
'location' in self.headers and
self.status_code in (codes.moved_permanently, codes.permanent_redirect)
)
@property
def next(self):
@@ -725,52 +750,58 @@ class Response(object):
try:
for chunk in self.raw.stream(chunk_size, decode_content=True):
yield chunk
except ProtocolError as e:
if self.headers.get('Transfer-Encoding') == 'chunked':
raise ChunkedEncodingError(e)
else:
raise ConnectionError(e)
except DecodeError as e:
raise ContentDecodingError(e)
except ReadTimeoutError as e:
raise ReadTimeout(e)
else:
# Standard file-like object.
while True:
chunk = self.raw.read(chunk_size)
if not chunk:
break
yield chunk
self._content_consumed = True
if self._content_consumed and isinstance(self._content, bool):
raise StreamConsumedError()
elif chunk_size is not None and not isinstance(chunk_size, int):
raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size))
raise TypeError(
"chunk_size must be an int, it is instead a %s." % type(chunk_size)
)
# simulate reading small chunks of the content
reused_chunks = iter_slices(self._content, chunk_size)
stream_chunks = generate()
chunks = reused_chunks if self._content_consumed else stream_chunks
if decode_unicode:
if self.encoding is None:
raise TypeError(
'encoding must be set before consuming streaming '
'responses'
'encoding must be set before consuming streaming ' 'responses'
)
# check encoding value here, don't wait for the generator to be
# consumed before raising an exception
codecs.lookup(self.encoding)
chunks = stream_decode_response_unicode(chunks, self)
return chunks
def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None):
def iter_lines(
self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None
):
"""Iterates over the response data, one line at a time. When
stream=True is set on the request, this avoids reading the
content at once into memory for large responses.
@@ -779,12 +810,11 @@ class Response(object):
"""
carriage_return = u'\r' if decode_unicode else b'\r'
line_feed = u'\n' if decode_unicode else b'\n'
pending = None
last_chunk_ends_with_cr = False
for chunk in self.iter_content(chunk_size=chunk_size,
decode_unicode=decode_unicode):
for chunk in self.iter_content(
chunk_size=chunk_size, decode_unicode=decode_unicode
):
# Skip any null responses: if there is pending data it is necessarily an
# incomplete chunk, so if we don't have more data we don't want to bother
# trying to get it. Unconsumed pending data will be yielded anyway in the
@@ -796,7 +826,6 @@ class Response(object):
if pending is not None:
chunk = pending + chunk
pending = None
# Either split on a line, or split on a specified delimiter
if delimiter:
lines = chunk.split(delimiter)
@@ -807,19 +836,20 @@ class Response(object):
# starts with '\n', they should be merged and treated as only
# *one* new line separator '\r\n' by splitlines().
# This rule only applies when splitlines() is used.
# The last chunk ends with '\r', so the '\n' at chunk[0]
# is just the second half of a '\r\n' pair rather than a
# new line break. Just skip it.
skip_first_char = last_chunk_ends_with_cr and chunk.startswith(line_feed)
skip_first_char = last_chunk_ends_with_cr and chunk.startswith(
line_feed
)
last_chunk_ends_with_cr = chunk.endswith(carriage_return)
if skip_first_char:
chunk = chunk[1:]
# it's possible that after stripping the '\n' then chunk becomes empty
if not chunk:
continue
lines = chunk.splitlines()
lines = chunk.splitlines()
# Calling `.split(delimiter)` will always end with whatever text
# remains beyond the delimiter, or '' if the delimiter is the end
# of the text. On the other hand, `.splitlines()` doesn't include
@@ -838,7 +868,6 @@ class Response(object):
incomplete_line = lines[-1] and lines[-1][-1] == chunk[-1]
if delimiter or incomplete_line:
pending = lines.pop()
for line in lines:
yield line
@@ -848,18 +877,18 @@ class Response(object):
@property
def content(self):
"""Content of the response, in bytes."""
if self._content is False:
# Read the contents.
if self._content_consumed:
raise RuntimeError(
'The content for this response was already consumed')
raise RuntimeError('The content for this response was already consumed')
if self.status_code == 0 or self.raw is None:
self._content = None
else:
self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()
self._content = bytes().join(
self.iter_content(CONTENT_CHUNK_SIZE)
) or bytes(
)
self._content_consumed = True
# don't need to release the connection; that's been handled by urllib3
# since we exhausted the data.
@@ -877,18 +906,15 @@ class Response(object):
non-HTTP knowledge to make a better guess at the encoding, you should
set ``r.encoding`` appropriately before accessing this property.
"""
# Try charset from content-type
content = None
encoding = self.encoding
if not self.content:
return str('')
# Fallback to auto-detected encoding.
if self.encoding is None:
encoding = self.apparent_encoding
# Decode unicode from given encoding.
try:
content = str(self.content, encoding, errors='replace')
@@ -900,7 +926,6 @@ class Response(object):
#
# So we try blindly encoding.
content = str(self.content, errors='replace')
return content
def json(self, **kwargs):
@@ -909,7 +934,6 @@ class Response(object):
:param \*\*kwargs: Optional arguments that ``json.loads`` takes.
:raises ValueError: If the response body does not contain valid json.
"""
if not self.encoding and self.content and len(self.content) > 3:
# No encoding set. JSON RFC 4627 section 3 states we should expect
# UTF-8, -16 or -32. Detect which one to use; If the detection or
@@ -918,9 +942,8 @@ class Response(object):
encoding = guess_json_utf(self.content)
if encoding is not None:
try:
return complexjson.loads(
self.content.decode(encoding), **kwargs
)
return complexjson.loads(self.content.decode(encoding), **kwargs)
except UnicodeDecodeError:
# Wrong UTF codec detected; usually because it's not UTF-8
# but some other 8-bit codec. This is an RFC violation,
@@ -932,25 +955,19 @@ class Response(object):
@property
def links(self):
"""Returns the parsed header links of the response, if any."""
header = self.headers.get('link')
# l = MultiDict()
l = {}
if header:
links = parse_header_links(header)
for link in links:
key = link.get('rel') or link.get('url')
l[key] = link
return l
def raise_for_status(self):
"""Raises stored :class:`HTTPError`, if one occurred.
Otherwise, returns the response object (self)."""
http_error_msg = ''
if isinstance(self.reason, bytes):
# We attempt to decode utf-8 first because some servers
@@ -963,13 +980,14 @@ class Response(object):
reason = self.reason.decode('iso-8859-1')
else:
reason = self.reason
if 400 <= self.status_code < 500:
http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url)
http_error_msg = u'%s Client Error: %s for url: %s' % (
self.status_code, reason, self.url
)
elif 500 <= self.status_code < 600:
http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url)
http_error_msg = u'%s Server Error: %s for url: %s' % (
self.status_code, reason, self.url
)
if http_error_msg:
raise HTTPError(http_error_msg, response=self)
@@ -983,7 +1001,6 @@ class Response(object):
"""
if not self._content_consumed:
self.raw.close()
release_conn = getattr(self.raw, 'release_conn', None)
if release_conn is not None:
release_conn()
requests/sessions.py
+110 -137
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests.session
~~~~~~~~~~~~~~~~
@@ -16,22 +15,36 @@ from datetime import timedelta
from .auth import _basic_auth_str
from .basics import cookielib, urljoin, urlparse, str
from .cookies import (
cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar,
merge_cookies, _copy_cookie_jar)
cookiejar_from_dict,
extract_cookies_to_jar,
RequestsCookieJar,
merge_cookies,
_copy_cookie_jar,
)
from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT
from .hooks import default_hooks, dispatch_hook
from ._internal_utils import to_native_string
from .utils import to_key_val_list, default_headers
from .exceptions import (
TooManyRedirects, InvalidScheme, ChunkedEncodingError,
ConnectionError, ContentDecodingError, InvalidHeader)
TooManyRedirects,
InvalidScheme,
ChunkedEncodingError,
ConnectionError,
ContentDecodingError,
InvalidHeader,
)
from .structures import CaseInsensitiveDict
from .adapters import HTTPAdapter
from .utils import (
requote_uri, get_environ_proxies, get_netrc_auth, should_bypass_proxies,
get_auth_from_url, is_valid_location, rewind_body
requote_uri,
get_environ_proxies,
get_netrc_auth,
should_bypass_proxies,
get_auth_from_url,
is_valid_location,
rewind_body,
)
from .status_codes import codes
@@ -54,7 +67,6 @@ def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
the explicit setting on that request, and the setting in the session. If a
setting is a dictionary, they will be merged together using `dict_class`.
"""
if session_setting is None:
return request_setting
@@ -63,20 +75,17 @@ def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
# Bypass if not a dictionary (e.g. verify)
if not (
isinstance(session_setting, Mapping) and
isinstance(request_setting, Mapping)
isinstance(session_setting, Mapping) and isinstance(request_setting, Mapping)
):
return request_setting
merged_setting = dict_class(to_key_val_list(session_setting))
merged_setting.update(to_key_val_list(request_setting))
# Remove keys that are set to None. Extract keys first to avoid altering
# the dictionary during iteration.
none_keys = [k for (k, v) in merged_setting.items() if v is None]
for key in none_keys:
del merged_setting[key]
return merged_setting
@@ -107,11 +116,12 @@ class SessionRedirectMixin(object):
# attribute.
if response.is_redirect:
if not is_valid_location(response):
raise InvalidHeader('Response contains multiple Location headers. '
'Unable to perform redirect.')
raise InvalidHeader(
'Response contains multiple Location headers. '
'Unable to perform redirect.'
)
location = response.headers['location']
# Currently the underlying http module on py3 decode headers
# in latin1, but empirical evidence suggests that latin1 is very
# rarely used with non-ASCII characters in HTTP headers.
@@ -120,43 +130,56 @@ class SessionRedirectMixin(object):
# To solve this, we re-encode the location in latin1.
location = location.encode('latin1')
return to_native_string(location, 'utf8')
return None
def resolve_redirects(self, response, request, stream=False, timeout=None,
verify=True, cert=None, proxies=None,
yield_requests=False, **adapter_kwargs):
def resolve_redirects(
self,
response,
request,
stream=False,
timeout=None,
verify=True,
cert=None,
proxies=None,
yield_requests=False,
**adapter_kwargs,
):
"""Given a Response, yields Responses until 'Location' header-based
redirection ceases, or the Session.max_redirects limit has been
reached.
"""
history = [response] # keep track of history; seed it with the original response
history = [
response
] # keep track of history; seed it with the original response
location_url = self.get_redirect_target(response)
while location_url:
prepared_request = request.copy()
try:
response.content # Consume socket so it can be released
except (ChunkedEncodingError, ConnectionError, ContentDecodingError, RuntimeError):
except (
ChunkedEncodingError,
ConnectionError,
ContentDecodingError,
RuntimeError,
):
response.raw.read(decode_content=False)
if len(response.history) >= self.max_redirects:
raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=response)
raise TooManyRedirects(
'Exceeded %s redirects.' % self.max_redirects, response=response
)
# Release the connection back into the pool.
response.close()
# Handle redirection without scheme (see: RFC 1808 Section 4)
if location_url.startswith('//'):
parsed_rurl = urlparse(response.url)
location_url = '%s:%s' % (to_native_string(parsed_rurl.scheme), location_url)
location_url = '%s:%s' % (
to_native_string(parsed_rurl.scheme), location_url
)
# The scheme should be lower case...
parsed = urlparse(location_url)
location_url = parsed.geturl()
# Facilitate relative 'location' headers, as allowed by RFC 7231.
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
# Compliant with RFC3986, we percent encode the url.
@@ -164,11 +187,8 @@ class SessionRedirectMixin(object):
location_url = urljoin(response.url, requote_uri(location_url))
else:
location_url = requote_uri(location_url)
prepared_request.url = to_native_string(location_url)
method_changed = self.rebuild_method(prepared_request, response)
# https://github.com/kennethreitz/requests/issues/2590
# If method is changed to GET we need to remove body and associated headers.
if method_changed and prepared_request.method == 'GET':
@@ -177,24 +197,20 @@ class SessionRedirectMixin(object):
for header in purged_headers:
prepared_request.headers.pop(header, None)
prepared_request.body = None
headers = prepared_request.headers
try:
del headers['Cookie']
except KeyError:
pass
# Extract any cookies sent on the response to the cookiejar
# in the new request. Because we've mutated our copied prepared
# request, use the old one that we haven't yet touched.
extract_cookies_to_jar(prepared_request._cookies, request, response.raw)
merge_cookies(prepared_request._cookies, self.cookies)
prepared_request.prepare_cookies(prepared_request._cookies)
# Rebuild auth and proxy information.
proxies = self.rebuild_proxies(prepared_request, proxies)
self.rebuild_auth(prepared_request, response)
# A failed tell() sets `_body_position` to `object()`. This non-None
# value ensures `rewindable` will be True, allowing us to raise an
# UnrewindableBodyError, instead of hanging the connection.
@@ -202,18 +218,15 @@ class SessionRedirectMixin(object):
prepared_request._body_position is not None and
('Content-Length' in headers or 'Transfer-Encoding' in headers)
)
# Attempt to rewind consumed file-like object.
if rewindable:
rewind_body(prepared_request)
# Override the original request.
request = prepared_request
if yield_requests:
yield request
else:
else:
response = self.send(
request,
stream=stream,
@@ -222,15 +235,13 @@ class SessionRedirectMixin(object):
cert=cert,
proxies=proxies,
allow_redirects=False,
**adapter_kwargs
**adapter_kwargs,
)
# copy our history tracker into the response
response.history = history[:]
# append the new response to the history tracker for the next iteration
history.append(response)
extract_cookies_to_jar(self.cookies, prepared_request, response.raw)
# extract redirect url, if any, for the next loop
location_url = self.get_redirect_target(response)
yield response
@@ -243,21 +254,17 @@ class SessionRedirectMixin(object):
"""
headers = prepared_request.headers
url = prepared_request.url
if 'Authorization' in headers:
# If we get redirected to a new host, we should strip out any
# authentication headers.
original_parsed = urlparse(response.request.url)
redirect_parsed = urlparse(url)
if (original_parsed.hostname != redirect_parsed.hostname):
del headers['Authorization']
# .netrc might have more auth for us on our new host.
new_auth = get_netrc_auth(url) if self.trust_env else None
if new_auth is not None:
prepared_request.prepare_auth(new_auth)
return
def rebuild_proxies(self, prepared_request, proxies):
@@ -278,27 +285,20 @@ class SessionRedirectMixin(object):
scheme = urlparse(url).scheme
new_proxies = proxies.copy()
no_proxy = proxies.get('no_proxy')
bypass_proxy = should_bypass_proxies(url, no_proxy=no_proxy)
if self.trust_env and not bypass_proxy:
environ_proxies = get_environ_proxies(url, no_proxy=no_proxy)
proxy = environ_proxies.get(scheme, environ_proxies.get('all'))
if proxy:
new_proxies.setdefault(scheme, proxy)
if 'Proxy-Authorization' in headers:
del headers['Proxy-Authorization']
try:
username, password = get_auth_from_url(new_proxies[scheme])
except KeyError:
username, password = None, None
if username and password:
headers['Proxy-Authorization'] = _basic_auth_str(username, password)
return new_proxies
def rebuild_method(self, prepared_request, response):
@@ -309,11 +309,9 @@ class SessionRedirectMixin(object):
:return: boolean expressing if the method changed during rebuild.
"""
method = original_method = prepared_request.method
# http://tools.ietf.org/html/rfc7231#section-6.4.4
if response.status_code == codes.see_other and method != 'HEAD':
method = 'GET'
# If a POST is responded to with a 301 or 302, turn it into a GET. This has
# become a common pattern in browsers and was introduced into later versions
# of HTTP RFCs. While some browsers transform other methods to GET, little of
@@ -321,7 +319,6 @@ class SessionRedirectMixin(object):
# which only supports POST->GET.
if response.status_code in (codes.found, codes.moved) and method == 'POST':
method = 'GET'
prepared_request.method = method
return method != original_method
@@ -344,63 +341,60 @@ class Session(SessionRedirectMixin):
>>> s.get('http://httpbin.org/get')
<Response [200]>
"""
__attrs__ = [
'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify',
'cert', 'prefetch', 'adapters', 'stream', 'trust_env',
'headers',
'cookies',
'auth',
'proxies',
'hooks',
'params',
'verify',
'cert',
'prefetch',
'adapters',
'stream',
'trust_env',
'max_redirects',
]
def __init__(self):
#: A case-insensitive dictionary of headers to be sent on each
# : A case-insensitive dictionary of headers to be sent on each
#: :class:`Request <Request>` sent from this
#: :class:`Session <Session>`.
self.headers = default_headers()
#: Default Authentication tuple or object to attach to
# : Default Authentication tuple or object to attach to
#: :class:`Request <Request>`.
self.auth = None
#: Dictionary mapping protocol or protocol and host to the URL of the proxy
# : Dictionary mapping protocol or protocol and host to the URL of the proxy
#: (e.g. {'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}) to
#: be used on each :class:`Request <Request>`.
self.proxies = {}
#: Event-handling hooks.
# : Event-handling hooks.
self.hooks = default_hooks()
#: Dictionary of querystring data to attach to each
# : Dictionary of querystring data to attach to each
#: :class:`Request <Request>`. The dictionary values may be lists for
#: representing multivalued query parameters.
self.params = {}
#: Stream response content default.
# : Stream response content default.
self.stream = False
#: SSL Verification default.
# : SSL Verification default.
self.verify = True
#: SSL client certificate default, if String, path to ssl client
# : SSL client certificate default, if String, path to ssl client
#: cert file (.pem). If Tuple, ('cert', 'key') pair.
self.cert = None
#: Maximum number of redirects allowed. If the request exceeds this
# : Maximum number of redirects allowed. If the request exceeds this
#: limit, a :class:`TooManyRedirects` exception is raised.
#: This defaults to requests.models.DEFAULT_REDIRECT_LIMIT, which is
#: 30.
self.max_redirects = DEFAULT_REDIRECT_LIMIT
#: Trust environment settings for proxy configuration, default
# : Trust environment settings for proxy configuration, default
#: authentication and similar.
self.trust_env = True
#: A CookieJar containing all currently outstanding cookies set on this
# : A CookieJar containing all currently outstanding cookies set on this
#: session. By default it is a
#: :class:`RequestsCookieJar <requests.cookies.RequestsCookieJar>`, but
#: may be any other ``cookielib.CookieJar`` compatible object.
self.cookies = cookiejar_from_dict({})
# Default connection adapters.
self.adapters = OrderedDict()
self.mount('https://', HTTPAdapter())
@@ -423,20 +417,16 @@ class Session(SessionRedirectMixin):
:rtype: requests.PreparedRequest
"""
cookies = request.cookies or {}
# Bootstrap CookieJar.
if not isinstance(cookies, cookielib.CookieJar):
cookies = cookiejar_from_dict(cookies)
# Merge with session cookies
session_cookies = _copy_cookie_jar(self.cookies)
merged_cookies = merge_cookies(session_cookies, cookies)
# Set environment's basic authentication if not explicitly set.
auth = request.auth
if self.trust_env and not auth and not self.auth:
auth = get_netrc_auth(request.url)
p = PreparedRequest()
p.prepare(
method=request.method.upper(),
@@ -444,7 +434,9 @@ class Session(SessionRedirectMixin):
files=request.files,
data=request.data,
json=request.json,
headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict),
headers=merge_setting(
request.headers, self.headers, dict_class=CaseInsensitiveDict
),
params=merge_setting(request.params, self.params),
auth=merge_setting(auth, self.auth),
cookies=merged_cookies,
@@ -452,10 +444,25 @@ class Session(SessionRedirectMixin):
)
return p
def request(self, method, url,
params=None, data=None, headers=None, cookies=None, files=None,
auth=None, timeout=None, allow_redirects=True, proxies=None,
hooks=None, stream=None, verify=None, cert=None, json=None):
def request(
self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=None,
cert=None,
json=None,
):
"""Constructs a :class:`Request <Request>`, prepares it, and sends it.
Returns :class:`Response <Response>` object.
@@ -506,21 +513,14 @@ class Session(SessionRedirectMixin):
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or {}
settings = self.merge_environment_settings(
prep.url, proxies, stream, verify, cert
)
# Send the request.
send_kwargs = {
'timeout': timeout,
'allow_redirects': allow_redirects,
}
send_kwargs = {'timeout': timeout, 'allow_redirects': allow_redirects}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
return resp
def get(self, url, **kwargs):
@@ -530,7 +530,6 @@ class Session(SessionRedirectMixin):
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
kwargs.setdefault('allow_redirects', True)
return self.request('GET', url, **kwargs)
@@ -541,7 +540,6 @@ class Session(SessionRedirectMixin):
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
kwargs.setdefault('allow_redirects', True)
return self.request('OPTIONS', url, **kwargs)
@@ -552,7 +550,6 @@ class Session(SessionRedirectMixin):
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
kwargs.setdefault('allow_redirects', False)
return self.request('HEAD', url, **kwargs)
@@ -565,7 +562,6 @@ class Session(SessionRedirectMixin):
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
return self.request('POST', url, data=data, json=json, **kwargs)
def put(self, url, data=None, **kwargs):
@@ -576,7 +572,6 @@ class Session(SessionRedirectMixin):
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
return self.request('PUT', url, data=data, **kwargs)
def patch(self, url, data=None, **kwargs):
@@ -587,7 +582,6 @@ class Session(SessionRedirectMixin):
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
return self.request('PATCH', url, data=data, **kwargs)
def delete(self, url, **kwargs):
@@ -597,7 +591,6 @@ class Session(SessionRedirectMixin):
:param \*\*kwargs: Optional arguments that ``request`` takes.
:rtype: requests.Response
"""
return self.request('DELETE', url, **kwargs)
def send(self, request, **kwargs):
@@ -611,7 +604,6 @@ class Session(SessionRedirectMixin):
kwargs.setdefault('verify', self.verify)
kwargs.setdefault('cert', self.cert)
kwargs.setdefault('proxies', self.proxies)
# It's possible that users might accidentally send a Request object.
# Guard against that specific failure case.
if isinstance(request, Request):
@@ -622,52 +614,40 @@ class Session(SessionRedirectMixin):
allow_redirects = kwargs.pop('allow_redirects', True)
stream = kwargs.get('stream')
hooks = request.hooks
# Get the appropriate adapter to use
adapter = self.get_adapter(url=request.url)
# Start time (approximately) of the request
start = preferred_clock()
# Send the request
r = adapter.send(request, **kwargs)
# Total elapsed time of the request (approximately)
elapsed = preferred_clock() - start
r.elapsed = timedelta(seconds=elapsed)
# Response manipulation hooks.
r = dispatch_hook('response', hooks, r, **kwargs)
# Persist cookies
if r.history:
# If the hooks create history then we want those cookies too
for resp in r.history:
extract_cookies_to_jar(self.cookies, resp.request, resp.raw)
extract_cookies_to_jar(self.cookies, request, r.raw)
# Redirect resolving generator.
gen = self.resolve_redirects(r, request, **kwargs)
# Resolve redirects, if allowed.
history = [resp for resp in gen] if allow_redirects else []
# If there is a history, replace ``r`` with the last response
if history:
r = history.pop()
# If redirects aren't being followed, store the response on the Request for Response.next().
if not allow_redirects:
try:
r._next = next(self.resolve_redirects(r, request, yield_requests=True, **kwargs))
r._next = next(
self.resolve_redirects(r, request, yield_requests=True, **kwargs)
)
except StopIteration:
pass
if not stream:
r.content
return r
def merge_environment_settings(self, url, proxies, stream, verify, cert):
@@ -687,10 +667,11 @@ class Session(SessionRedirectMixin):
# Look for requests environment configuration and be compatible
# with cURL.
if verify is True or verify is None:
verify = (os.environ.get('REQUESTS_CA_BUNDLE') or
os.environ.get('CURL_CA_BUNDLE') or
verify)
verify = (
os.environ.get('REQUESTS_CA_BUNDLE') or
os.environ.get('CURL_CA_BUNDLE') or
verify
)
# Now we handle proxies.
# Proxies need to be built up backwards. This is because None values
# can delete proxy information, which can then be re-added by a more
@@ -699,17 +680,12 @@ class Session(SessionRedirectMixin):
no_proxy = proxies.get('no_proxy') if proxies is not None else None
if no_proxy is None:
no_proxy = self.proxies.get('no_proxy')
env_proxies = {}
if self.trust_env:
env_proxies = get_environ_proxies(url, no_proxy=no_proxy) or {}
new_proxies = merge_setting(self.proxies, env_proxies)
proxies = merge_setting(proxies, new_proxies)
return {'verify': verify, 'proxies': proxies, 'stream': stream,
'cert': cert}
return {'verify': verify, 'proxies': proxies, 'stream': stream, 'cert': cert}
def get_adapter(self, url):
"""
@@ -718,7 +694,6 @@ class Session(SessionRedirectMixin):
:rtype: requests.adapters.BaseAdapter
"""
for (prefix, adapter) in self.adapters.items():
if url.lower().startswith(prefix):
return adapter
@@ -737,7 +712,6 @@ class Session(SessionRedirectMixin):
"""
self.adapters[prefix] = adapter
keys_to_move = [k for k in self.adapters if len(k) < len(prefix)]
for key in keys_to_move:
self.adapters[key] = self.adapters.pop(key)
@@ -756,5 +730,4 @@ def session():
:rtype: Session
"""
return Session()
requests/status_codes.py
+6 -11
View File
@@ -1,9 +1,7 @@
# -*- coding: utf-8 -*-
from .structures import LookupDict
_codes = {
# Informational.
100: ('continue',),
101: ('switching_protocols',),
@@ -20,7 +18,6 @@ _codes = {
207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'),
208: ('already_reported',),
226: ('im_used',),
# Redirection.
300: ('multiple_choices',),
301: ('moved_permanently', 'moved', '\\o-'),
@@ -30,9 +27,8 @@ _codes = {
305: ('use_proxy',),
306: ('switch_proxy',),
307: ('temporary_redirect', 'temporary_moved', 'temporary'),
308: ('permanent_redirect',
'resume_incomplete', 'resume',), # These 2 to be removed in 3.0
308: ('permanent_redirect', 'resume_incomplete', 'resume'),
# These 2 to be removed in 3.0
# Client Error.
400: ('bad_request', 'bad'),
401: ('unauthorized',),
@@ -50,7 +46,9 @@ _codes = {
413: ('request_entity_too_large',),
414: ('request_uri_too_large',),
415: ('unsupported_media_type', 'unsupported_media', 'media_type'),
416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'),
416: (
'requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'
),
417: ('expectation_failed',),
418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'),
421: ('misdirected_request',),
@@ -67,7 +65,6 @@ _codes = {
450: ('blocked_by_windows_parental_controls', 'parental_controls'),
451: ('unavailable_for_legal_reasons', 'legal_reasons'),
499: ('client_closed_request',),
# Server Error.
500: ('internal_server_error', 'server_error', '/o\\', '✗'),
501: ('not_implemented',),
@@ -81,11 +78,9 @@ _codes = {
510: ('not_extended',),
511: ('network_authentication_required', 'network_auth', 'network_authentication'),
}
codes = LookupDict(name='status_codes')
for code, titles in _codes.items():
for title in titles: # type: ignore
for title in titles: # type: ignore
setattr(codes, title, code)
if not title.startswith(('\\', '/')):
setattr(codes, title.upper(), code)
requests/structures.py
+3 -7
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests.structures
~~~~~~~~~~~~~~~~~~~
@@ -62,20 +61,18 @@ class CaseInsensitiveDict(collections.MutableMapping):
def lower_items(self):
"""Like iteritems(), but with all lowercase keys."""
return (
(lowerkey, keyval[1])
for (lowerkey, keyval)
in self._store.items()
)
return ((lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items())
def __eq__(self, other):
if isinstance(other, collections.Mapping):
other = CaseInsensitiveDict(other)
else:
return NotImplemented
# Compare insensitively
return dict(self.lower_items()) == dict(other.lower_items())
# Copy is required
def copy(self):
return CaseInsensitiveDict(self._store.values())
@@ -96,7 +93,6 @@ class LookupDict(dict):
def __getitem__(self, key):
# We allow fall-through here, so values default to None
return self.__dict__.get(key, None)
def __iter__(self):
requests/types.py
+29 -16
View File
@@ -1,24 +1,36 @@
from typing import (
Callable, Optional, Union, Any, Iterable, List, Mapping, MutableMapping,
Tuple, IO, Text, Type, Dict
Callable,
Optional,
Union,
Any,
Iterable,
List,
Mapping,
MutableMapping,
Tuple,
IO,
Text,
Type,
Dict,
)
from . import auth
from .import auth
from .models import Response, PreparedRequest
from .cookies import RequestsCookieJar
from .sessions import Session
_ParamsMappingValueType = Union[str, bytes, int, float, Iterable[Union[str, bytes, int, float]]]
_ParamsMappingValueType = Union[
str, bytes, int, float, Iterable[Union[str, bytes, int, float]]
]
Params = Optional[
Union[
Mapping[
Union[str, bytes, int, float], _ParamsMappingValueType],
Union[str, bytes],
Tuple[Union[str, bytes, int, float], _ParamsMappingValueType],
Mapping[str, _ParamsMappingValueType],
Mapping[bytes, _ParamsMappingValueType],
Mapping[int, _ParamsMappingValueType],
Mapping[float, _ParamsMappingValueType]
Mapping[Union[str, bytes, int, float], _ParamsMappingValueType],
Union[str, bytes],
Tuple[Union[str, bytes, int, float], _ParamsMappingValueType],
Mapping[str, _ParamsMappingValueType],
Mapping[bytes, _ParamsMappingValueType],
Mapping[int, _ParamsMappingValueType],
Mapping[float, _ParamsMappingValueType],
]
]
Data = Union[
@@ -29,16 +41,17 @@ Data = Union[
MutableMapping[Text, str],
MutableMapping[Text, Text],
Iterable[Tuple[str, str]],
IO
IO,
]
_Hook = Callable[[Response], Any]
Method = str
URL = str
Headers = Optional[Union[None, MutableMapping[Text, Text]]]
Cookies = Optional[Union[None, RequestsCookieJar, MutableMapping[Text, Text]]]
Files = Optional[MutableMapping[Text, IO]]
Auth = Union[None, Tuple[Text, Text], auth.AuthBase, Callable[[PreparedRequest], PreparedRequest]]
Auth = Union[
None, Tuple[Text, Text], auth.AuthBase, Callable[[PreparedRequest], PreparedRequest]
]
Timeout = Union[None, float, Tuple[float, float]]
AllowRedirects = Optional[bool]
Proxies = Optional[MutableMapping[Text, Text]]
@@ -47,4 +60,4 @@ Stream = Optional[bool]
Verify = Union[None, bool, Text]
Cert = Union[Text, Tuple[Text, Text]]
JSON = Optional[MutableMapping]
Help = Dict
Help = Dict
requests/utils.py
+129 -114
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""
requests.utils
~~~~~~~~~~~~~~
@@ -21,38 +20,49 @@ import struct
import warnings
from .__version__ import __version__
from . import certs
from .import certs
# to_native_string is unused here, but imported here for backwards compatibility
from ._internal_utils import to_native_string
from .basics import parse_http_list as _parse_list_header
from .basics import (
quote, urlparse, bytes, str, unquote, getproxies,
proxy_bypass, urlunparse, basestring, integer_types,
proxy_bypass_environment, getproxies_environment)
quote,
urlparse,
bytes,
str,
unquote,
getproxies,
proxy_bypass,
urlunparse,
basestring,
integer_types,
proxy_bypass_environment,
getproxies_environment,
)
from .cookies import cookiejar_from_dict
from .structures import CaseInsensitiveDict
from .exceptions import (
InvalidURL, InvalidHeader, FileModeWarning, UnrewindableBodyError)
InvalidURL, InvalidHeader, FileModeWarning, UnrewindableBodyError
)
NETRC_FILES = ('.netrc', '_netrc')
DEFAULT_CA_BUNDLE_PATH = certs.where()
if platform.system() == 'Windows':
# provide a proxy_bypass version on Windows without DNS lookups
# provide a proxy_bypass version on Windows without DNS lookups
def proxy_bypass_registry(host):
import winreg
try:
internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
proxyEnable = winreg.QueryValueEx(internetSettings,
'ProxyEnable')[0]
proxyOverride = winreg.QueryValueEx(internetSettings,
'ProxyOverride')[0]
internetSettings = winreg.OpenKey(
winreg.HKEY_CURRENT_USER,
r'Software\Microsoft\Windows\CurrentVersion\Internet Settings',
)
proxyEnable = winreg.QueryValueEx(internetSettings, 'ProxyEnable')[0]
proxyOverride = winreg.QueryValueEx(internetSettings, 'ProxyOverride')[0]
except OSError:
return False
if not proxyEnable or not proxyOverride:
return False
@@ -65,11 +75,13 @@ if platform.system() == 'Windows':
if test == '<local>':
if '.' not in host:
return True
test = test.replace(".", r"\.") # mask dots
test = test.replace("*", r".*") # change glob sequence
test = test.replace("?", r".") # change glob char
test = test.replace(".", r"\.") # mask dots
test = test.replace("*", r".*") # change glob sequence
test = test.replace("?", r".") # change glob char
if re.match(test, host, re.I):
return True
return False
def proxy_bypass(host): # noqa
@@ -80,29 +92,25 @@ if platform.system() == 'Windows':
"""
if getproxies_environment():
return proxy_bypass_environment(host)
else:
return proxy_bypass_registry(host)
def dict_to_sequence(d):
"""Returns an internal sequence dictionary update."""
if hasattr(d, 'items'):
d = d.items()
return d
def super_len(o):
total_length = None
current_position = 0
if hasattr(o, '__len__'):
total_length = len(o)
elif hasattr(o, 'len'):
total_length = o.len
elif hasattr(o, 'fileno'):
try:
fileno = o.fileno()
@@ -110,20 +118,20 @@ def super_len(o):
pass
else:
total_length = os.fstat(fileno).st_size
# Having used fstat to determine the file length, we need to
# confirm that this file was opened up in binary mode.
if 'b' not in o.mode:
warnings.warn((
"Requests has determined the content-length for this "
"request using the binary size of the file: however, the "
"file has been opened in text mode (i.e. without the 'b' "
"flag in the mode). This may lead to an incorrect "
"content-length. In Requests 3.0, support will be removed "
"for files in text mode."),
FileModeWarning
warnings.warn(
(
"Requests has determined the content-length for this "
"request using the binary size of the file: however, the "
"file has been opened in text mode (i.e. without the 'b' "
"flag in the mode). This may lead to an incorrect "
"content-length. In Requests 3.0, support will be removed "
"for files in text mode."
),
FileModeWarning,
)
if hasattr(o, 'tell'):
try:
current_position = o.tell()
@@ -141,27 +149,22 @@ def super_len(o):
# seek to end of file
o.seek(0, 2)
total_length = o.tell()
# seek back to current position to support
# partially read file-like objects
o.seek(current_position or 0)
except (OSError, IOError):
total_length = 0
if total_length is None:
total_length = 0
return max(0, total_length - current_position)
def get_netrc_auth(url, raise_errors=False):
"""Returns the Requests tuple auth for a given url from netrc."""
try:
from netrc import netrc, NetrcParseError
netrc_path = None
for f in NETRC_FILES:
try:
loc = os.path.expanduser('~/{0}'.format(f))
@@ -180,20 +183,19 @@ def get_netrc_auth(url, raise_errors=False):
return
ri = urlparse(url)
# Strip port numbers from netloc. This weird `if...encode`` dance is
# used for Python 3.2, which doesn't support unicode literals.
splitstr = b':'
if isinstance(url, str):
splitstr = splitstr.decode('ascii')
host = ri.netloc.split(splitstr)[0]
try:
_netrc = netrc(netrc_path).authenticators(host)
if _netrc:
# Return with login / password
login_i = (0 if _netrc[0] else 1)
return (_netrc[login_i], _netrc[2])
except (NetrcParseError, IOError):
# If there was a parsing error or a permissions issue reading the file,
# we'll just skip netrc auth unless explicitly asked to raise errors.
@@ -208,8 +210,7 @@ def get_netrc_auth(url, raise_errors=False):
def guess_filename(obj):
"""Tries to guess the filename of the given object."""
name = getattr(obj, 'name', None)
if (name and isinstance(name, basestring) and name[0] != '<' and
name[-1] != '>'):
if (name and isinstance(name, basestring) and name[0] != '<' and name[-1] != '>'):
return os.path.basename(name)
@@ -261,10 +262,11 @@ def to_key_val_list(value):
if isinstance(value, collections.Mapping):
value = value.items()
return list(value)
# From mitsuhiko/werkzeug (used with permission).
def parse_list_header(value):
"""Parse lists as described by RFC 2068 Section 2.
@@ -297,6 +299,8 @@ def parse_list_header(value):
return result
# From mitsuhiko/werkzeug (used with permission).
def parse_dict_header(value):
"""Parse lists of key, value pairs as described by RFC 2068 Section 2 and
@@ -325,6 +329,7 @@ def parse_dict_header(value):
if '=' not in item:
result[item] = None
continue
name, value = item.split('=', 1)
if value[:1] == value[-1:] == '"':
value = unquote_header_value(value[1:-1])
@@ -332,6 +337,8 @@ def parse_dict_header(value):
return result
# From mitsuhiko/werkzeug (used with permission).
def unquote_header_value(value, is_filename=False):
r"""Unquotes a header value. (Reversal of :func:`quote_header_value`).
@@ -347,7 +354,6 @@ def unquote_header_value(value, is_filename=False):
# probably some other browsers as well. IE for example is
# uploading files with "C:\foo\bar.txt" as filename
value = value[1:-1]
# if this is a filename and the starting characters look like
# a UNC path, then just return the value without quotes. Using the
# replace sequence below on a UNC path has the effect of turning
@@ -355,6 +361,7 @@ def unquote_header_value(value, is_filename=False):
# _fix_ie_filename() doesn't work correctly. See #458.
if not is_filename or value[:2] != '\\\\':
return value.replace('\\\\', '\\').replace('\\"', '"')
return value
@@ -364,12 +371,9 @@ def dict_from_cookiejar(cj):
:param cj: CookieJar object to extract cookies from.
:rtype: dict
"""
cookie_dict = {}
for cookie in cj:
cookie_dict[cookie.name] = cookie.value
return cookie_dict
@@ -380,7 +384,6 @@ def add_dict_to_cookiejar(cj, cookie_dict):
:param cookie_dict: Dict of key/values to insert into CookieJar.
:rtype: CookieJar
"""
return cookiejar_from_dict(cookie_dict, cj)
@@ -389,19 +392,22 @@ def get_encodings_from_content(content):
:param content: bytestring to extract encodings from.
"""
warnings.warn((
'In requests 3.0, get_encodings_from_content will be removed. For '
'more information, please see the discussion on issue #2266. (This'
' warning should only appear once.)'),
DeprecationWarning)
warnings.warn(
(
'In requests 3.0, get_encodings_from_content will be removed. For '
'more information, please see the discussion on issue #2266. (This'
' warning should only appear once.)'
),
DeprecationWarning,
)
charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)
xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]')
return (charset_re.findall(content) +
pragma_re.findall(content) +
xml_re.findall(content))
return (
charset_re.findall(content) +
pragma_re.findall(content) +
xml_re.findall(content)
)
def get_encoding_from_headers(headers):
@@ -410,14 +416,11 @@ def get_encoding_from_headers(headers):
:param headers: dictionary to extract encoding from.
:rtype: str
"""
content_type = headers.get('content-type')
if not content_type:
return None
content_type, params = cgi.parse_header(content_type)
if 'charset' in params:
return params['charset'].strip("'\"")
@@ -427,12 +430,12 @@ def get_encoding_from_headers(headers):
def stream_decode_response_unicode(iterator, r):
"""Stream decodes a iterator."""
decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace')
for chunk in iterator:
rv = decoder.decode(chunk)
if rv:
yield rv
rv = decoder.decode(b'', final=True)
if rv:
yield rv
@@ -444,7 +447,8 @@ def iter_slices(string, slice_length):
if slice_length is None or slice_length <= 0:
slice_length = len(string)
while pos < len(string):
yield string[pos:pos + slice_length]
yield string[pos: pos + slice_length]
pos += slice_length
@@ -460,33 +464,35 @@ def get_unicode_from_response(r):
:rtype: str
"""
warnings.warn((
'In requests 3.0, get_unicode_from_response will be removed. For '
'more information, please see the discussion on issue #2266. (This'
' warning should only appear once.)'),
DeprecationWarning)
warnings.warn(
(
'In requests 3.0, get_unicode_from_response will be removed. For '
'more information, please see the discussion on issue #2266. (This'
' warning should only appear once.)'
),
DeprecationWarning,
)
tried_encodings = []
# Try charset from content-type
encoding = get_encoding_from_headers(r.headers)
if encoding:
try:
return str(r.content, encoding)
except UnicodeError:
tried_encodings.append(encoding)
# Fall back:
try:
return str(r.content, encoding, errors='replace')
except TypeError:
return r.content
# The unreserved URI characters (RFC 3986)
UNRESERVED_SET = frozenset(
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~")
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~"
)
def unquote_unreserved(uri):
@@ -495,12 +501,14 @@ def unquote_unreserved(uri):
:rtype: str
"""
# This convert function is used to optionally convert the output of `chr`.
# In Python 3, `chr` returns a unicode string, while in Python 2 it returns
# a bytestring. Here we deal with that by optionally converting.
def convert(is_bytes, c):
if is_bytes:
return c.encode('ascii')
else:
return c
@@ -511,7 +519,6 @@ def unquote_unreserved(uri):
if is_bytes:
splitchar = splitchar.encode('ascii')
base = base.encode('ascii')
parts = uri.split(splitchar)
for i in range(1, len(parts)):
h = parts[i][0:2]
@@ -545,6 +552,7 @@ def requote_uri(uri):
# Then quote only illegal characters (do not quote reserved,
# unreserved, or '%')
return quote(unquote_unreserved(uri), safe=safe_with_percent)
except InvalidURL:
# We couldn't unquote the given URI, so let's try quoting it, but
# there may be unquoted '%'s in the URI. We need to make sure they're
@@ -564,7 +572,7 @@ def address_in_network(ip, net):
netaddr, bits = net.split('/')
netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0]
network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask
return (ipaddr & netmask) == (network & netmask)
return ( ipaddr & netmask) == ( network & netmask)
def dotted_netmask(mask):
@@ -586,6 +594,7 @@ def is_ipv4_address(string_ip):
socket.inet_aton(string_ip)
except socket.error:
return False
return True
@@ -608,8 +617,10 @@ def is_valid_cidr(string_network):
socket.inet_aton(string_network.split('/')[0])
except socket.error:
return False
else:
return False
return True
@@ -627,6 +638,7 @@ def set_environ(env_name, value):
os.environ[env_name] = value
try:
yield
finally:
if value_changed:
if old_value is None:
@@ -642,31 +654,28 @@ def should_bypass_proxies(url, no_proxy):
:rtype: bool
"""
get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
# First check whether no_proxy is defined. If it is, check that the URL
# we're getting isn't in the no_proxy list.
no_proxy_arg = no_proxy
if no_proxy is None:
no_proxy = get_proxy('no_proxy')
netloc = urlparse(url).netloc
if no_proxy:
# We need to check whether we match here. We need to see if we match
# the end of the netloc, both with and without the port.
no_proxy = (
host for host in no_proxy.replace(' ', '').split(',') if host
)
no_proxy = (host for host in no_proxy.replace(' ', '').split(',') if host)
ip = netloc.split(':')[0]
if is_ipv4_address(ip):
for proxy_ip in no_proxy:
if is_valid_cidr(proxy_ip):
if address_in_network(ip, proxy_ip):
return True
elif ip == proxy_ip:
# If no_proxy ip was defined in plain IP notation instead of cidr notation &
# matches the IP of the index
return True
else:
for host in no_proxy:
if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
@@ -686,6 +695,7 @@ def get_environ_proxies(url, no_proxy=None):
"""
if should_bypass_proxies(url, no_proxy=no_proxy):
return {}
else:
return getproxies()
@@ -729,12 +739,14 @@ def default_headers():
"""
:rtype: requests.structures.CaseInsensitiveDict
"""
return CaseInsensitiveDict({
'User-Agent': default_user_agent(),
'Accept-Encoding': ', '.join(('gzip', 'deflate')),
'Accept': '*/*',
'Connection': 'keep-alive',
})
return CaseInsensitiveDict(
{
'User-Agent': default_user_agent(),
'Accept-Encoding': ', '.join(('gzip', 'deflate')),
'Accept': '*/*',
'Connection': 'keep-alive',
}
)
def parse_header_links(value):
@@ -744,11 +756,8 @@ def parse_header_links(value):
:rtype: list
"""
links = []
replace_chars = ' \'"'
value = value.strip(replace_chars)
if not value:
return links
@@ -758,9 +767,7 @@ def parse_header_links(value):
url, params = val.split(';', 1)
except ValueError:
url, params = val, ''
link = {'url': url.strip('<> \'"')}
for param in params.split(';'):
try:
key, value = param.split('=')
@@ -768,9 +775,7 @@ def parse_header_links(value):
break
link[key.strip(replace_chars)] = value.strip(replace_chars)
links.append(link)
return links
@@ -783,6 +788,7 @@ def is_valid_location(response):
getlist = getattr(headers, 'getlist', None)
if getlist is not None:
return len(getlist('location')) <= 1
# If response.raw isn't urllib3-like we can't reliably check this
return True
@@ -802,26 +808,34 @@ def guess_json_utf(data):
# determine the encoding. Also detect a BOM, if present.
sample = data[:4]
if sample in (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE):
return 'utf-32' # BOM included
return 'utf-32' # BOM included
if sample[:3] == codecs.BOM_UTF8:
return 'utf-8-sig' # BOM included, MS style (discouraged)
if sample[:2] in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE):
return 'utf-16' # BOM included
return 'utf-16' # BOM included
nullcount = sample.count(_null)
if nullcount == 0:
return 'utf-8'
if nullcount == 2:
if sample[::2] == _null2: # 1st and 3rd are null
if sample[::2] == _null2: # 1st and 3rd are null
return 'utf-16-be'
if sample[1::2] == _null2: # 2nd and 4th are null
return 'utf-16-le'
# Did not detect 2 valid UTF-16 ascii-range characters
# Did not detect 2 valid UTF-16 ascii-range characters
if nullcount == 3:
if sample[:3] == _null3:
return 'utf-32-be'
if sample[1:] == _null3:
return 'utf-32-le'
# Did not detect a valid UTF-32 ascii-range character
# Did not detect a valid UTF-32 ascii-range character
return None
@@ -832,13 +846,11 @@ def prepend_scheme_if_needed(url, new_scheme):
:rtype: str
"""
scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme)
# urlparse is a finicky beast, and sometimes decides that there isn't a
# netloc present. Assume that it's being over-cautious, and switch netloc
# and path if urlparse decided there was no netloc.
if not netloc:
netloc, path = path, netloc
return urlunparse((scheme, netloc, path, params, query, fragment))
@@ -849,12 +861,10 @@ def get_auth_from_url(url):
:rtype: (str,str)
"""
parsed = urlparse(url)
try:
auth = (unquote(parsed.username), unquote(parsed.password))
except (AttributeError, TypeError):
auth = ('', '')
return auth
@@ -871,17 +881,21 @@ def check_header_validity(header):
:param header: tuple, in the format (name, value).
"""
name, value = header
if isinstance(value, bytes):
pat = _CLEAN_HEADER_REGEX_BYTE
else:
pat = _CLEAN_HEADER_REGEX_STR
try:
if not pat.match(value):
raise InvalidHeader("Invalid return character or leading space in header: %s" % name)
raise InvalidHeader(
"Invalid return character or leading space in header: %s" % name
)
except TypeError:
raise InvalidHeader("Value for header {%s: %s} must be of type str or "
"bytes, not %s" % (name, value, type(value)))
raise InvalidHeader(
"Value for header {%s: %s} must be of type str or "
"bytes, not %s" % (name, value, type(value))
)
def urldefragauth(url):
@@ -891,13 +905,10 @@ def urldefragauth(url):
:rtype: str
"""
scheme, netloc, path, params, query, fragment = urlparse(url)
# see func:`prepend_scheme_if_needed`
if not netloc:
netloc, path = path, netloc
netloc = netloc.rsplit('@', 1)[-1]
return urlunparse((scheme, netloc, path, params, query, ''))
@@ -906,12 +917,16 @@ def rewind_body(prepared_request):
so it can be read again on redirect.
"""
body_seek = getattr(prepared_request.body, 'seek', None)
if body_seek is not None and isinstance(prepared_request._body_position, integer_types):
if body_seek is not None and isinstance(
prepared_request._body_position, integer_types
):
try:
body_seek(prepared_request._body_position)
except (IOError, OSError):
raise UnrewindableBodyError("An error occurred when rewinding request "
"body for redirect.")
raise UnrewindableBodyError(
"An error occurred when rewinding request " "body for redirect."
)
else:
raise UnrewindableBodyError("Unable to rewind request body for redirect.")
setup.py
+14 -15
View File
@@ -1,6 +1,5 @@
#!/usr/bin/env python
# Learn more: https://github.com/kennethreitz/setup.py
import os
import sys
@@ -55,27 +54,30 @@ if sys.argv[-1] == 'publish':
os.system('python setup.py sdist bdist_wheel')
os.system('twine upload dist/*')
sys.exit()
packages = ['requests']
requires = [
'chardet>=3.0.2,<3.1.0',
'idna>=2.5,<2.7',
'urllib3>=1.21.1,<1.23',
'certifi>=2017.4.17'
'certifi>=2017.4.17',
]
test_requirements = [
'pytest-httpbin==0.0.7',
'pytest-cov',
'pytest-mock',
'pytest-xdist',
'PySocks>=1.5.6, !=1.5.7',
'pytest>=2.8.0',
'pytest-mypy',
'mypy==0.540',
]
test_requirements = ['pytest-httpbin==0.0.7', 'pytest-cov', 'pytest-mock', 'pytest-xdist', 'PySocks>=1.5.6, !=1.5.7', 'pytest>=2.8.0', 'pytest-mypy', 'mypy==0.540']
about = {}
with open(os.path.join(here, 'requests', '__version__.py'), 'r', 'utf-8') as f:
exec(f.read(), about)
exec (f.read(), about)
with open('README.rst', 'r', 'utf-8') as f:
readme = f.read()
with open('HISTORY.rst', 'r', 'utf-8') as f:
history = f.read()
setup(
name=about['__title__'],
version=about['__version__'],
@@ -102,12 +104,9 @@ setup(
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: Implementation :: CPython',
'Programming Language :: Python :: Implementation :: PyPy'
'Programming Language :: Python :: Implementation :: PyPy',
),
cmdclass={
'test': PyTest,
'mypy': MyPyTest
},
cmdclass={'test': PyTest, 'mypy': MyPyTest},
tests_require=test_requirements,
extras_require={
'security': ['pyOpenSSL>=0.14', 'cryptography>=1.3.4', 'idna>=2.0.0'],
tests/__init__.py
-1
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
"""Requests test package initialisation."""
import warnings
tests/compat.py
+1 -1
View File
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
import io as StringIO
def u(s):
return s
tests/conftest.py
-1
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
import pytest
from urllib.parse import urljoin
tests/test_help.py
+3 -3
View File
@@ -1,5 +1,4 @@
# -*- encoding: utf-8
import sys
import pytest
@@ -7,7 +6,7 @@ import pytest
from requests.help import info
@pytest.mark.skipif(sys.version_info[:2] != (2,6), reason="Only run on Python 2.6")
@pytest.mark.skipif(sys.version_info[:2] != (2, 6), reason="Only run on Python 2.6")
def test_system_ssl_py26():
"""OPENSSL_VERSION_NUMBER isn't provided in Python 2.6, verify we don't
blow up in this case.
@@ -15,13 +14,14 @@ def test_system_ssl_py26():
assert info()['system_ssl'] == {'version': ''}
@pytest.mark.skipif(sys.version_info < (2,7), reason="Only run on Python 2.7+")
@pytest.mark.skipif(sys.version_info < (2, 7), reason="Only run on Python 2.7+")
def test_system_ssl():
"""Verify we're actually setting system_ssl when it should be available."""
assert info()['system_ssl']['version'] != ''
class VersionedPackage(object):
def __init__(self, version):
self.__version__ = version
tests/test_hooks.py
+1 -5
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
import pytest
from requests import hooks
@@ -10,10 +9,7 @@ def hook(value):
@pytest.mark.parametrize(
'hooks_list, result', (
(hook, 'ata'),
([hook, lambda x: None, hook], 'ta'),
)
'hooks_list, result', ((hook, 'ata'), ([hook, lambda x: None, hook], 'ta'))
)
def test_hooks(hooks_list, result):
assert hooks.dispatch_hook('response', {'response': hooks_list}, 'Data') == result
tests/test_lowlevel.py
+39 -61
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
import pytest
import threading
import requests
@@ -14,23 +13,20 @@ def test_chunked_upload():
close_server = threading.Event()
server = Server.basic_response_server(wait_to_close_event=close_server)
data = iter([b'a', b'b', b'c'])
with server as (host, port):
url = 'http://{0}:{1}/'.format(host, port)
r = requests.post(url, data=data, stream=True)
close_server.set() # release server block
assert r.status_code == 200
assert r.request.headers['Transfer-Encoding'] == 'chunked'
def test_incorrect_content_length():
"""Test ConnectionError raised for incomplete responses"""
close_server = threading.Event()
server = Server.text_response_server(
"HTTP/1.1 200 OK\r\n" +
"Content-Length: 50\r\n\r\n" +
"Hello World."
)
"HTTP/1.1 200 OK\r\n" + "Content-Length: 50\r\n\r\n" + "Hello World."
)
with server as (host, port):
url = 'http://{0}:{1}/'.format(host, port)
r = requests.Request('GET', url).prepare()
@@ -47,23 +43,22 @@ def test_digestauth_401_count_reset_on_redirect():
See https://github.com/requests/requests/issues/1979.
"""
text_401 = (b'HTTP/1.1 401 UNAUTHORIZED\r\n'
b'Content-Length: 0\r\n'
b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"'
b', opaque="372825293d1c26955496c80ed6426e9e", '
b'realm="me@kennethreitz.com", qop=auth\r\n\r\n')
text_302 = (b'HTTP/1.1 302 FOUND\r\n'
b'Content-Length: 0\r\n'
b'Location: /\r\n\r\n')
text_200 = (b'HTTP/1.1 200 OK\r\n'
b'Content-Length: 0\r\n\r\n')
expected_digest = (b'Authorization: Digest username="user", '
b'realm="me@kennethreitz.com", '
b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"')
text_401 = (
b'HTTP/1.1 401 UNAUTHORIZED\r\n'
b'Content-Length: 0\r\n'
b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"'
b', opaque="372825293d1c26955496c80ed6426e9e", '
b'realm="me@kennethreitz.com", qop=auth\r\n\r\n'
)
text_302 = (
b'HTTP/1.1 302 FOUND\r\n' b'Content-Length: 0\r\n' b'Location: /\r\n\r\n'
)
text_200 = (b'HTTP/1.1 200 OK\r\n' b'Content-Length: 0\r\n\r\n')
expected_digest = (
b'Authorization: Digest username="user", '
b'realm="me@kennethreitz.com", '
b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"'
)
auth = requests.auth.HTTPDigestAuth('user', 'pass')
def digest_response_handler(sock):
@@ -71,28 +66,23 @@ def test_digestauth_401_count_reset_on_redirect():
request_content = consume_socket_content(sock, timeout=0.5)
assert request_content.startswith(b"GET / HTTP/1.1")
sock.send(text_401)
# Verify we receive an Authorization header in response, then redirect.
request_content = consume_socket_content(sock, timeout=0.5)
assert expected_digest in request_content
sock.send(text_302)
# Verify Authorization isn't sent to the redirected host,
# then send another challenge.
request_content = consume_socket_content(sock, timeout=0.5)
assert b'Authorization:' not in request_content
sock.send(text_401)
# Verify Authorization is sent correctly again, and return 200 OK.
request_content = consume_socket_content(sock, timeout=0.5)
assert expected_digest in request_content
sock.send(text_200)
return request_content
close_server = threading.Event()
server = Server(digest_response_handler, wait_to_close_event=close_server)
with server as (host, port):
url = 'http://{0}:{1}/'.format(host, port)
r = requests.get(url, auth=auth)
@@ -110,16 +100,18 @@ def test_digestauth_401_only_sent_once():
"""Ensure we correctly respond to a 401 challenge once, and then
stop responding if challenged again.
"""
text_401 = (b'HTTP/1.1 401 UNAUTHORIZED\r\n'
b'Content-Length: 0\r\n'
b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"'
b', opaque="372825293d1c26955496c80ed6426e9e", '
b'realm="me@kennethreitz.com", qop=auth\r\n\r\n')
expected_digest = (b'Authorization: Digest username="user", '
b'realm="me@kennethreitz.com", '
b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"')
text_401 = (
b'HTTP/1.1 401 UNAUTHORIZED\r\n'
b'Content-Length: 0\r\n'
b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"'
b', opaque="372825293d1c26955496c80ed6426e9e", '
b'realm="me@kennethreitz.com", qop=auth\r\n\r\n'
)
expected_digest = (
b'Authorization: Digest username="user", '
b'realm="me@kennethreitz.com", '
b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"'
)
auth = requests.auth.HTTPDigestAuth('user', 'pass')
def digest_failed_response_handler(sock):
@@ -127,22 +119,18 @@ def test_digestauth_401_only_sent_once():
request_content = consume_socket_content(sock, timeout=0.5)
assert request_content.startswith(b"GET / HTTP/1.1")
sock.send(text_401)
# Verify we receive an Authorization header in response, then
# challenge again.
request_content = consume_socket_content(sock, timeout=0.5)
assert expected_digest in request_content
sock.send(text_401)
# Verify the client didn't respond to second challenge.
request_content = consume_socket_content(sock, timeout=0.5)
assert request_content == b''
return request_content
close_server = threading.Event()
server = Server(digest_failed_response_handler, wait_to_close_event=close_server)
with server as (host, port):
url = 'http://{0}:{1}/'.format(host, port)
r = requests.get(url, auth=auth)
@@ -157,12 +145,13 @@ def test_digestauth_only_on_4xx():
See https://github.com/requests/requests/issues/3772.
"""
text_200_chal = (b'HTTP/1.1 200 OK\r\n'
b'Content-Length: 0\r\n'
b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"'
b', opaque="372825293d1c26955496c80ed6426e9e", '
b'realm="me@kennethreitz.com", qop=auth\r\n\r\n')
text_200_chal = (
b'HTTP/1.1 200 OK\r\n'
b'Content-Length: 0\r\n'
b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"'
b', opaque="372825293d1c26955496c80ed6426e9e", '
b'realm="me@kennethreitz.com", qop=auth\r\n\r\n'
)
auth = requests.auth.HTTPDigestAuth('user', 'pass')
def digest_response_handler(sock):
@@ -170,16 +159,13 @@ def test_digestauth_only_on_4xx():
request_content = consume_socket_content(sock, timeout=0.5)
assert request_content.startswith(b"GET / HTTP/1.1")
sock.send(text_200_chal)
# Verify the client didn't respond with auth.
request_content = consume_socket_content(sock, timeout=0.5)
assert request_content == b''
return request_content
close_server = threading.Event()
server = Server(digest_response_handler, wait_to_close_event=close_server)
with server as (host, port):
url = 'http://{0}:{1}/'.format(host, port)
r = requests.get(url, auth=auth)
@@ -190,16 +176,12 @@ def test_digestauth_only_on_4xx():
_schemes_by_var_prefix = [
('http', ['http']),
('https', ['https']),
('all', ['http', 'https']),
('http', ['http']), ('https', ['https']), ('all', ['http', 'https'])
]
_proxy_combos = []
for prefix, schemes in _schemes_by_var_prefix:
for scheme in schemes:
_proxy_combos.append(("{0}_proxy".format(prefix), scheme))
_proxy_combos += [(var.upper(), scheme) for var, scheme in _proxy_combos]
@@ -214,10 +196,8 @@ def test_use_proxy_from_environment(httpbin, var, scheme):
# fake proxy's lack of response will cause a ConnectionError
with pytest.raises(requests.exceptions.ConnectionError):
requests.get(url)
# the fake proxy received a request
assert len(fake_proxy.handler_results) == 1
# it had actual content (not checking for SOCKS protocol for now)
assert len(fake_proxy.handler_results[0]) > 0
@@ -241,7 +221,6 @@ def test_redirect_rfc1808_to_non_ascii_location():
close_server = threading.Event()
server = Server(redirect_resp_handler, wait_to_close_event=close_server)
with server as (host, port):
url = u'http://{0}:{1}'.format(host, port)
r = requests.get(url=url, allow_redirects=True)
@@ -250,5 +229,4 @@ def test_redirect_rfc1808_to_non_ascii_location():
assert r.history[0].status_code == 301
assert redirect_request[0].startswith(b'GET /' + expected_path + b' HTTP/1.1')
assert r.url == u'{0}/{1}'.format(url, expected_path.decode('ascii'))
close_server.set()
tests/test_requests.py
+354 -465
View File
File diff suppressed because it is too large Load Diff
tests/test_structures.py
+9 -12
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
import pytest
from requests.structures import CaseInsensitiveDict, LookupDict
@@ -16,7 +15,9 @@ class TestCaseInsensitiveDict:
def test_list(self):
assert list(self.case_insensitive_dict) == ['Accept']
possible_keys = pytest.mark.parametrize('key', ('accept', 'ACCEPT', 'aCcEpT', 'Accept'))
possible_keys = pytest.mark.parametrize(
'key', ('accept', 'ACCEPT', 'aCcEpT', 'Accept')
)
@possible_keys
def test_getitem(self, key):
@@ -28,7 +29,9 @@ class TestCaseInsensitiveDict:
assert key not in self.case_insensitive_dict
def test_lower_items(self):
assert list(self.case_insensitive_dict.lower_items()) == [('accept', 'application/json')]
assert list(self.case_insensitive_dict.lower_items()) == [
('accept', 'application/json')
]
def test_repr(self):
assert repr(self.case_insensitive_dict) == "{'Accept': 'application/json'}"
@@ -39,11 +42,8 @@ class TestCaseInsensitiveDict:
assert copy == self.case_insensitive_dict
@pytest.mark.parametrize(
'other, result', (
({'AccePT': 'application/json'}, True),
({}, False),
(None, False)
)
'other, result',
(({'AccePT': 'application/json'}, True), ({}, False), (None, False)),
)
def test_instance_equality(self, other, result):
assert (self.case_insensitive_dict == other) is result
@@ -61,10 +61,7 @@ class TestLookupDict:
assert repr(self.lookup_dict) == "<lookup 'test'>"
get_item_parameters = pytest.mark.parametrize(
'key, value', (
('bad_gateway', 502),
('not_a_key', None)
)
'key, value', (('bad_gateway', 502), ('not_a_key', None))
)
@get_item_parameters
tests/test_testserver.py
+9 -26
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
import threading
import socket
import time
@@ -34,9 +33,7 @@ class TestTestServer:
with Server.basic_response_server() as (host, port):
sock = socket.socket()
sock.connect((host, port))
sock.close()
with pytest.raises(socket.error):
new_sock = socket.socket()
new_sock.connect((host, port))
@@ -44,14 +41,10 @@ class TestTestServer:
def test_text_response(self):
"""the text_response_server sends the given text"""
server = Server.text_response_server(
"HTTP/1.1 200 OK\r\n" +
"Content-Length: 6\r\n" +
"\r\nroflol"
"HTTP/1.1 200 OK\r\n" + "Content-Length: 6\r\n" + "\r\nroflol"
)
with server as (host, port):
r = requests.get('http://{0}:{1}'.format(host, port))
assert r.status_code == 200
assert r.text == u'roflol'
assert r.headers['Content-Length'] == '6'
@@ -67,8 +60,9 @@ class TestTestServer:
def test_basic_waiting_server(self):
"""the server waits for the block_server event to be set before closing"""
block_server = threading.Event()
with Server.basic_response_server(wait_to_close_event=block_server) as (host, port):
with Server.basic_response_server(wait_to_close_event=block_server) as (
host, port
):
sock = socket.socket()
sock.connect((host, port))
sock.sendall(b'send something')
@@ -79,15 +73,12 @@ class TestTestServer:
def test_multiple_requests(self):
"""multiple requests can be served"""
requests_to_handle = 5
server = Server.basic_response_server(requests_to_handle=requests_to_handle)
with server as (host, port):
server_url = 'http://{0}:{1}'.format(host, port)
for _ in range(requests_to_handle):
r = requests.get(server_url)
assert r.status_code == 200
# the (n+1)th request fails
with pytest.raises(requests.exceptions.ConnectionError):
r = requests.get(server_url)
@@ -99,47 +90,39 @@ class TestTestServer:
server = Server.basic_response_server(requests_to_handle=2)
first_request = b'put your hands up in the air'
second_request = b'put your hand down in the floor'
with server as address:
sock1 = socket.socket()
sock2 = socket.socket()
sock1.connect(address)
sock1.sendall(first_request)
sock1.close()
sock2.connect(address)
sock2.sendall(second_request)
sock2.close()
assert server.handler_results[0] == first_request
assert server.handler_results[1] == second_request
def test_requests_after_timeout_are_not_received(self):
"""the basic response handler times out when receiving requests"""
server = Server.basic_response_server(request_timeout=1)
with server as address:
sock = socket.socket()
sock.connect(address)
time.sleep(1.5)
sock.sendall(b'hehehe, not received')
sock.close()
assert server.handler_results[0] == b''
def test_request_recovery_with_bigger_timeout(self):
"""a biggest timeout can be specified"""
server = Server.basic_response_server(request_timeout=3)
data = b'bananadine'
with server as address:
sock = socket.socket()
sock.connect(address)
time.sleep(1.5)
sock.sendall(data)
sock.close()
assert server.handler_results[0] == data
def test_server_finishes_on_error(self):
@@ -151,16 +134,16 @@ class TestTestServer:
assert len(server.handler_results) == 0
# if the server thread fails to finish, the test suite will hang
# and get killed by the jenkins timeout.
# if the server thread fails to finish, the test suite will hang
# and get killed by the jenkins timeout.
def test_server_finishes_when_no_connections(self):
"""the server thread exits even if there are no connections"""
server = Server.basic_response_server()
with server:
pass
assert len(server.handler_results) == 0
# if the server thread fails to finish, the test suite will hang
# and get killed by the jenkins timeout.
# if the server thread fails to finish, the test suite will hang
# and get killed by the jenkins timeout.
tests/test_utils.py
+235 -249
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
import os
import copy
from io import BytesIO
@@ -9,16 +8,31 @@ from requests import basics
from requests.cookies import RequestsCookieJar
from requests.structures import CaseInsensitiveDict
from requests.utils import (
address_in_network, dotted_netmask,
get_auth_from_url, get_encoding_from_headers,
get_encodings_from_content, get_environ_proxies,
guess_filename, guess_json_utf, is_ipv4_address,
is_valid_cidr, iter_slices, parse_dict_header,
parse_header_links, prepend_scheme_if_needed,
requote_uri, select_proxy, should_bypass_proxies, super_len,
to_key_val_list, to_native_string,
unquote_header_value, unquote_unreserved,
urldefragauth, add_dict_to_cookiejar, set_environ
address_in_network,
dotted_netmask,
get_auth_from_url,
get_encoding_from_headers,
get_encodings_from_content,
get_environ_proxies,
guess_filename,
guess_json_utf,
is_ipv4_address,
is_valid_cidr,
iter_slices,
parse_dict_header,
parse_header_links,
prepend_scheme_if_needed,
requote_uri,
select_proxy,
should_bypass_proxies,
super_len,
to_key_val_list,
to_native_string,
unquote_header_value,
unquote_unreserved,
urldefragauth,
add_dict_to_cookiejar,
set_environ,
)
from requests._internal_utils import unicode_is_ascii
@@ -28,10 +42,8 @@ from .compat import StringIO
class TestSuperLen:
@pytest.mark.parametrize(
'stream, value', (
(StringIO.StringIO, 'Test'),
(BytesIO, b'Test')
))
'stream, value', ((StringIO.StringIO, 'Test'), (BytesIO, b'Test'))
)
def test_io_streams(self, stream, value):
"""Ensures that we properly deal with different kinds of IO streams."""
assert super_len(stream()) == 0
@@ -46,7 +58,9 @@ class TestSuperLen:
@pytest.mark.parametrize('error', [IOError, OSError])
def test_super_len_handles_files_raising_weird_errors_in_tell(self, error):
"""If tell() raises errors, assume the cursor is at position zero."""
class BoomFile(object):
def __len__(self):
return 5
@@ -58,7 +72,9 @@ class TestSuperLen:
@pytest.mark.parametrize('error', [IOError, OSError])
def test_super_len_tell_ioerror(self, error):
"""Ensure that if tell gives an IOError super_len doesn't fail"""
class NoLenBoomFile(object):
def tell(self):
raise error()
@@ -70,11 +86,7 @@ class TestSuperLen:
def test_string(self):
assert super_len('Test') == 4
@pytest.mark.parametrize(
'mode, warnings_num', (
('r', 1),
('rb', 0),
))
@pytest.mark.parametrize('mode, warnings_num', (('r', 1), ('rb', 0)))
def test_file(self, tmpdir, mode, warnings_num, recwarn):
file_obj = tmpdir.join('test.txt')
file_obj.write('Test')
@@ -83,12 +95,14 @@ class TestSuperLen:
assert len(recwarn) == warnings_num
def test_super_len_with__len__(self):
foo = [1,2,3,4]
foo = [1, 2, 3, 4]
len_foo = super_len(foo)
assert len_foo == 4
def test_super_len_with_no__len__(self):
class LenFile(object):
def __init__(self):
self.len = 5
@@ -114,12 +128,14 @@ class TestSuperLen:
class TestToKeyValList:
@pytest.mark.parametrize(
'value, expected', (
'value, expected',
(
([('key', 'val')], [('key', 'val')]),
((('key', 'val'), ), [('key', 'val')]),
((('key', 'val'),), [('key', 'val')]),
({'key': 'val'}, [('key', 'val')]),
(None, None)
))
(None, None),
),
)
def test_valid(self, value, expected):
assert to_key_val_list(value) == expected
@@ -131,13 +147,15 @@ class TestToKeyValList:
class TestUnquoteHeaderValue:
@pytest.mark.parametrize(
'value, expected', (
'value, expected',
(
(None, None),
('Test', 'Test'),
('"Test"', 'Test'),
('"Test\\\\"', 'Test\\'),
('"\\\\Comp\\Res"', '\\Comp\\Res'),
))
),
)
def test_valid(self, value, expected):
assert unquote_header_value(value) == expected
@@ -152,46 +170,48 @@ class TestGetEnvironProxies:
@pytest.fixture(autouse=True, params=['no_proxy', 'NO_PROXY'])
def no_proxy(self, request, monkeypatch):
monkeypatch.setenv(request.param, '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
monkeypatch.setenv(
request.param, '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1'
)
@pytest.mark.parametrize(
'url', (
'url',
(
'http://192.168.0.1:5000/',
'http://192.168.0.1/',
'http://172.16.1.1/',
'http://172.16.1.1:5000/',
'http://localhost.localdomain:5000/v1.0/',
))
),
)
def test_bypass(self, url):
assert get_environ_proxies(url, no_proxy=None) == {}
@pytest.mark.parametrize(
'url', (
'http://192.168.1.1:5000/',
'http://192.168.1.1/',
'http://www.requests.com/',
))
'url',
('http://192.168.1.1:5000/', 'http://192.168.1.1/', 'http://www.requests.com/'),
)
def test_not_bypass(self, url):
assert get_environ_proxies(url, no_proxy=None) != {}
@pytest.mark.parametrize(
'url', (
'http://192.168.1.1:5000/',
'http://192.168.1.1/',
'http://www.requests.com/',
))
'url',
('http://192.168.1.1:5000/', 'http://192.168.1.1/', 'http://www.requests.com/'),
)
def test_bypass_no_proxy_keyword(self, url):
no_proxy = '192.168.1.1,requests.com'
assert get_environ_proxies(url, no_proxy=no_proxy) == {}
@pytest.mark.parametrize(
'url', (
'url',
(
'http://192.168.0.1:5000/',
'http://192.168.0.1/',
'http://172.16.1.1/',
'http://172.16.1.1:5000/',
'http://localhost.localdomain:5000/v1.0/',
))
),
)
def test_not_bypass_no_proxy_keyword(self, url, monkeypatch):
# This is testing that the 'no_proxy' argument overrides the
# environment variable 'no_proxy'
@@ -216,13 +236,15 @@ class TestIsValidCIDR:
assert is_valid_cidr('192.168.1.0/24')
@pytest.mark.parametrize(
'value', (
'value',
(
'8.8.8.8',
'192.168.1.0/a',
'192.168.1.0/128',
'192.168.1.0/-1',
'192.168.1.999/24',
))
),
)
def test_invalid(self, value):
assert not is_valid_cidr(value)
@@ -238,17 +260,14 @@ class TestAddressInNetwork:
class TestGuessFilename:
@pytest.mark.parametrize(
'value', (1, type('Fake', (object,), {'name': 1})()),
)
@pytest.mark.parametrize('value', (1, type('Fake', (object,), {'name': 1})()))
def test_guess_filename_invalid(self, value):
assert guess_filename(value) is None
@pytest.mark.parametrize(
'value, expected_type', (
(b'value', basics.bytes),
(b'value'.decode('utf-8'), basics.str)
))
'value, expected_type',
((b'value', basics.bytes), (b'value'.decode('utf-8'), basics.str)),
)
def test_guess_filename_valid(self, value, expected_type):
obj = type('Fake', (object,), {'name': value})()
result = guess_filename(obj)
@@ -263,16 +282,13 @@ class TestContentEncodingDetection:
assert not len(encodings)
@pytest.mark.parametrize(
'content', (
# HTML5 meta charset attribute
'<meta charset="UTF-8">',
# HTML4 pragma directive
'<meta http-equiv="Content-type" content="text/html;charset=UTF-8">',
# XHTML 1.x served with text/html MIME type
'<meta http-equiv="Content-type" content="text/html;charset=UTF-8" />',
# XHTML 1.x served as XML
'<?xml version="1.0" encoding="UTF-8"?>',
))
'content',
('<meta charset="UTF-8">', '<meta http-equiv="Content-type" content="text/html;charset=UTF-8">', '<meta http-equiv="Content-type" content="text/html;charset=UTF-8" />', '<?xml version="1.0" encoding="UTF-8"?>'),
# HTML5 meta charset attribute
# HTML4 pragma directive
# XHTML 1.x served with text/html MIME type
# XHTML 1.x served as XML
)
def test_pragmas(self, content):
encodings = get_encodings_from_content(content)
assert len(encodings) == 1
@@ -283,17 +299,26 @@ class TestContentEncodingDetection:
<?xml version="1.0" encoding="XML"?>
<meta charset="HTML5">
<meta http-equiv="Content-type" content="text/html;charset=HTML4" />
'''.strip()
'''.strip(
)
assert get_encodings_from_content(content) == ['HTML5', 'HTML4', 'XML']
class TestGuessJSONUTF:
@pytest.mark.parametrize(
'encoding', (
'utf-32', 'utf-8-sig', 'utf-16', 'utf-8', 'utf-16-be', 'utf-16-le',
'utf-32-be', 'utf-32-le'
))
'encoding',
(
'utf-32',
'utf-8-sig',
'utf-16',
'utf-8',
'utf-16-be',
'utf-16-le',
'utf-32-be',
'utf-32-le',
),
)
def test_encoded(self, encoding):
data = '{}'.encode(encoding)
assert guess_json_utf(data) == encoding
@@ -302,12 +327,14 @@ class TestGuessJSONUTF:
assert guess_json_utf(b'\x00\x00\x00\x00') is None
@pytest.mark.parametrize(
('encoding', 'expected'), (
('encoding', 'expected'),
(
('utf-16-be', 'utf-16'),
('utf-16-le', 'utf-16'),
('utf-32-be', 'utf-32'),
('utf-32-le', 'utf-32')
))
('utf-32-le', 'utf-32'),
),
)
def test_guess_by_bom(self, encoding, expected):
data = u'\ufeff{}'.encode(encoding)
assert guess_json_utf(data) == expected
@@ -319,156 +346,119 @@ ENCODED_PASSWORD = basics.quote(PASSWORD, '')
@pytest.mark.parametrize(
'url, auth', (
'url, auth',
(
(
'http://' + ENCODED_USER + ':' + ENCODED_PASSWORD + '@' +
'http://' +
ENCODED_USER +
':' +
ENCODED_PASSWORD +
'@' +
'request.com/url.html#test',
(USER, PASSWORD)
),
(
'http://user:pass@complex.url.com/path?query=yes',
('user', 'pass')
(USER, PASSWORD),
),
('http://user:pass@complex.url.com/path?query=yes', ('user', 'pass')),
(
'http://user:pass%20pass@complex.url.com/path?query=yes',
('user', 'pass pass')
),
(
'http://user:pass pass@complex.url.com/path?query=yes',
('user', 'pass pass')
('user', 'pass pass'),
),
('http://user:pass pass@complex.url.com/path?query=yes', ('user', 'pass pass')),
(
'http://user%25user:pass@complex.url.com/path?query=yes',
('user%user', 'pass')
('user%user', 'pass'),
),
(
'http://user:pass%23pass@complex.url.com/path?query=yes',
('user', 'pass#pass')
('user', 'pass#pass'),
),
(
'http://complex.url.com/path?query=yes',
('', '')
),
))
('http://complex.url.com/path?query=yes', ('', '')),
),
)
def test_get_auth_from_url(url, auth):
assert get_auth_from_url(url) == auth
@pytest.mark.parametrize(
'uri, expected', (
(
# Ensure requoting doesn't break expectations
'http://example.com/fiz?buz=%25ppicture',
'http://example.com/fiz?buz=%25ppicture',
),
(
# Ensure we handle unquoted percent signs in redirects
'http://example.com/fiz?buz=%ppicture',
'http://example.com/fiz?buz=%25ppicture',
),
))
'uri, expected',
(('http://example.com/fiz?buz=%25ppicture', 'http://example.com/fiz?buz=%25ppicture'), ('http://example.com/fiz?buz=%ppicture', 'http://example.com/fiz?buz=%25ppicture')),
# Ensure requoting doesn't break expectations
# Ensure we handle unquoted percent signs in redirects
)
def test_requote_uri_with_unquoted_percents(uri, expected):
"""See: https://github.com/requests/requests/issues/2356"""
assert requote_uri(uri) == expected
@pytest.mark.parametrize(
'uri, expected', (
(
# Illegal bytes
'http://example.com/?a=%--',
'http://example.com/?a=%--',
),
(
# Reserved characters
'http://example.com/?a=%300',
'http://example.com/?a=00',
)
))
'uri, expected',
(('http://example.com/?a=%--', 'http://example.com/?a=%--'), ('http://example.com/?a=%300', 'http://example.com/?a=00')),
# Illegal bytes
# Reserved characters
)
def test_unquote_unreserved(uri, expected):
assert unquote_unreserved(uri) == expected
@pytest.mark.parametrize(
'mask, expected', (
(8, '255.0.0.0'),
(24, '255.255.255.0'),
(25, '255.255.255.128'),
))
'mask, expected', ((8, '255.0.0.0'), (24, '255.255.255.0'), (25, '255.255.255.128'))
)
def test_dotted_netmask(mask, expected):
assert dotted_netmask(mask) == expected
http_proxies = {'http': 'http://http.proxy',
'http://some.host': 'http://some.host.proxy'}
all_proxies = {'all': 'socks5://http.proxy',
'all://some.host': 'socks5://some.host.proxy'}
mixed_proxies = {'http': 'http://http.proxy',
'http://some.host': 'http://some.host.proxy',
'all': 'socks5://http.proxy'}
http_proxies = {
'http': 'http://http.proxy', 'http://some.host': 'http://some.host.proxy'
}
all_proxies = {
'all': 'socks5://http.proxy', 'all://some.host': 'socks5://some.host.proxy'
}
mixed_proxies = {
'http': 'http://http.proxy',
'http://some.host': 'http://some.host.proxy',
'all': 'socks5://http.proxy',
}
@pytest.mark.parametrize(
'url, expected, proxies', (
('hTTp://u:p@Some.Host/path', 'http://some.host.proxy', http_proxies),
('hTTp://u:p@Other.Host/path', 'http://http.proxy', http_proxies),
('hTTp:///path', 'http://http.proxy', http_proxies),
('hTTps://Other.Host', None, http_proxies),
('file:///etc/motd', None, http_proxies),
('hTTp://u:p@Some.Host/path', 'socks5://some.host.proxy', all_proxies),
('hTTp://u:p@Other.Host/path', 'socks5://http.proxy', all_proxies),
('hTTp:///path', 'socks5://http.proxy', all_proxies),
('hTTps://Other.Host', 'socks5://http.proxy', all_proxies),
('http://u:p@other.host/path', 'http://http.proxy', mixed_proxies),
('http://u:p@some.host/path', 'http://some.host.proxy', mixed_proxies),
('https://u:p@other.host/path', 'socks5://http.proxy', mixed_proxies),
('https://u:p@some.host/path', 'socks5://http.proxy', mixed_proxies),
('https://', 'socks5://http.proxy', mixed_proxies),
# XXX: unsure whether this is reasonable behavior
('file:///etc/motd', 'socks5://http.proxy', all_proxies),
))
'url, expected, proxies',
(('hTTp://u:p@Some.Host/path', 'http://some.host.proxy', http_proxies), ('hTTp://u:p@Other.Host/path', 'http://http.proxy', http_proxies), ('hTTp:///path', 'http://http.proxy', http_proxies), ('hTTps://Other.Host', None, http_proxies), ('file:///etc/motd', None, http_proxies), ('hTTp://u:p@Some.Host/path', 'socks5://some.host.proxy', all_proxies), ('hTTp://u:p@Other.Host/path', 'socks5://http.proxy', all_proxies), ('hTTp:///path', 'socks5://http.proxy', all_proxies), ('hTTps://Other.Host', 'socks5://http.proxy', all_proxies), ('http://u:p@other.host/path', 'http://http.proxy', mixed_proxies), ('http://u:p@some.host/path', 'http://some.host.proxy', mixed_proxies), ('https://u:p@other.host/path', 'socks5://http.proxy', mixed_proxies), ('https://u:p@some.host/path', 'socks5://http.proxy', mixed_proxies), ('https://', 'socks5://http.proxy', mixed_proxies), ('file:///etc/motd', 'socks5://http.proxy', all_proxies)),
# XXX: unsure whether this is reasonable behavior
)
def test_select_proxies(url, expected, proxies):
"""Make sure we can select per-host proxies correctly."""
assert select_proxy(url, proxies) == expected
@pytest.mark.parametrize(
'value, expected', (
'value, expected',
(
('foo="is a fish", bar="as well"', {'foo': 'is a fish', 'bar': 'as well'}),
('key_without_value', {'key_without_value': None})
))
('key_without_value', {'key_without_value': None}),
),
)
def test_parse_dict_header(value, expected):
assert parse_dict_header(value) == expected
@pytest.mark.parametrize(
'value, expected', (
(
CaseInsensitiveDict(),
None
),
'value, expected',
(
(CaseInsensitiveDict(), None),
(
CaseInsensitiveDict({'content-type': 'application/json; charset=utf-8'}),
'utf-8'
'utf-8',
),
(
CaseInsensitiveDict({'content-type': 'text/plain'}),
'ISO-8859-1'
),
))
(CaseInsensitiveDict({'content-type': 'text/plain'}), 'ISO-8859-1'),
),
)
def test_get_encoding_from_headers(value, expected):
assert get_encoding_from_headers(value) == expected
@pytest.mark.parametrize(
'value, length', (
('', 0),
('T', 1),
('Test', 4),
('Cont', 0),
('Other', -5),
('Content', None),
))
'value, length',
(('', 0), ('T', 1), ('Test', 4), ('Cont', 0), ('Other', -5), ('Content', None)),
)
def test_iter_slices(value, length):
if length is None or (length <= 0 and len(value) > 0):
# Reads all content at once
@@ -478,127 +468,119 @@ def test_iter_slices(value, length):
@pytest.mark.parametrize(
'value, expected', (
'value, expected',
(
(
'<http:/.../front.jpeg>; rel=front; type="image/jpeg"',
[{'url': 'http:/.../front.jpeg', 'rel': 'front', 'type': 'image/jpeg'}]
),
(
'<http:/.../front.jpeg>',
[{'url': 'http:/.../front.jpeg'}]
),
(
'<http:/.../front.jpeg>;',
[{'url': 'http:/.../front.jpeg'}]
[{'url': 'http:/.../front.jpeg', 'rel': 'front', 'type': 'image/jpeg'}],
),
('<http:/.../front.jpeg>', [{'url': 'http:/.../front.jpeg'}]),
('<http:/.../front.jpeg>;', [{'url': 'http:/.../front.jpeg'}]),
(
'<http:/.../front.jpeg>; type="image/jpeg",<http://.../back.jpeg>;',
[
{'url': 'http:/.../front.jpeg', 'type': 'image/jpeg'},
{'url': 'http://.../back.jpeg'}
]
{'url': 'http://.../back.jpeg'},
],
),
(
'',
[]
),
))
('', []),
),
)
def test_parse_header_links(value, expected):
assert parse_header_links(value) == expected
@pytest.mark.parametrize(
'value, expected', (
'value, expected',
(
('example.com/path', 'http://example.com/path'),
('//example.com/path', 'http://example.com/path'),
))
),
)
def test_prepend_scheme_if_needed(value, expected):
assert prepend_scheme_if_needed(value, 'http') == expected
@pytest.mark.parametrize(
'value, expected', (
('T', 'T'),
(b'T', 'T'),
(u'T', 'T'),
))
@pytest.mark.parametrize('value, expected', (('T', 'T'), (b'T', 'T'), (u'T', 'T')))
def test_to_native_string(value, expected):
assert to_native_string(value) == expected
@pytest.mark.parametrize(
'url, expected', (
'url, expected',
(
('http://u:p@example.com/path?a=1#test', 'http://example.com/path?a=1'),
('http://example.com/path', 'http://example.com/path'),
('//u:p@example.com/path', '//example.com/path'),
('//example.com/path', '//example.com/path'),
('example.com/path', '//example.com/path'),
('scheme:u:p@example.com/path', 'scheme://example.com/path'),
))
),
)
def test_urldefragauth(url, expected):
assert urldefragauth(url) == expected
@pytest.mark.parametrize(
'url, expected', (
('http://192.168.0.1:5000/', True),
('http://192.168.0.1/', True),
('http://172.16.1.1/', True),
('http://172.16.1.1:5000/', True),
('http://localhost.localdomain:5000/v1.0/', True),
('http://172.16.1.12/', False),
('http://172.16.1.12:5000/', False),
('http://google.com:5000/v1.0/', False),
))
'url, expected',
(
('http://192.168.0.1:5000/', True),
('http://192.168.0.1/', True),
('http://172.16.1.1/', True),
('http://172.16.1.1:5000/', True),
('http://localhost.localdomain:5000/v1.0/', True),
('http://172.16.1.12/', False),
('http://172.16.1.12:5000/', False),
('http://google.com:5000/v1.0/', False),
),
)
def test_should_bypass_proxies(url, expected, monkeypatch):
"""Tests for function should_bypass_proxies to check if proxy
can be bypassed or not
"""
monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
monkeypatch.setenv(
'no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1'
)
monkeypatch.setenv(
'NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1'
)
assert should_bypass_proxies(url, no_proxy=None) == expected
@pytest.mark.parametrize(
'cookiejar', (
basics.cookielib.CookieJar(),
RequestsCookieJar()
))
'cookiejar', (basics.cookielib.CookieJar(), RequestsCookieJar())
)
def test_add_dict_to_cookiejar(cookiejar):
"""Ensure add_dict_to_cookiejar works for
non-RequestsCookieJar CookieJars
"""
cookiedict = {'test': 'cookies',
'good': 'cookies'}
cookiedict = {'test': 'cookies', 'good': 'cookies'}
cj = add_dict_to_cookiejar(cookiejar, cookiedict)
cookies = {cookie.name: cookie.value for cookie in cj}
assert cookiedict == cookies
@pytest.mark.parametrize(
'value, expected', (
(u'test', True),
(u'æíöû', False),
(u'ジェーピーニック', False),
)
'value, expected', ((u'test', True), (u'æíöû', False), (u'ジェーピーニック', False))
)
def test_unicode_is_ascii(value, expected):
assert unicode_is_ascii(value) is expected
@pytest.mark.parametrize(
'url, expected', (
('http://192.168.0.1:5000/', True),
('http://192.168.0.1/', True),
('http://172.16.1.1/', True),
('http://172.16.1.1:5000/', True),
('http://localhost.localdomain:5000/v1.0/', True),
('http://172.16.1.12/', False),
('http://172.16.1.12:5000/', False),
('http://google.com:5000/v1.0/', False),
))
def test_should_bypass_proxies_no_proxy(
url, expected, monkeypatch):
'url, expected',
(
('http://192.168.0.1:5000/', True),
('http://192.168.0.1/', True),
('http://172.16.1.1/', True),
('http://172.16.1.1:5000/', True),
('http://localhost.localdomain:5000/v1.0/', True),
('http://172.16.1.12/', False),
('http://172.16.1.12:5000/', False),
('http://google.com:5000/v1.0/', False),
),
)
def test_should_bypass_proxies_no_proxy(url, expected, monkeypatch):
"""Tests for function should_bypass_proxies to check if proxy
can be bypassed or not using the 'no_proxy' argument
"""
@@ -609,20 +591,21 @@ def test_should_bypass_proxies_no_proxy(
@pytest.mark.skipif(os.name != 'nt', reason='Test only on Windows')
@pytest.mark.parametrize(
'url, expected, override', (
('http://192.168.0.1:5000/', True, None),
('http://192.168.0.1/', True, None),
('http://172.16.1.1/', True, None),
('http://172.16.1.1:5000/', True, None),
('http://localhost.localdomain:5000/v1.0/', True, None),
('http://172.16.1.22/', False, None),
('http://172.16.1.22:5000/', False, None),
('http://google.com:5000/v1.0/', False, None),
('http://mylocalhostname:5000/v1.0/', True, '<local>'),
('http://192.168.0.1/', False, ''),
))
def test_should_bypass_proxies_win_registry(url, expected, override,
monkeypatch):
'url, expected, override',
(
('http://192.168.0.1:5000/', True, None),
('http://192.168.0.1/', True, None),
('http://172.16.1.1/', True, None),
('http://172.16.1.1:5000/', True, None),
('http://localhost.localdomain:5000/v1.0/', True, None),
('http://172.16.1.22/', False, None),
('http://172.16.1.22:5000/', False, None),
('http://google.com:5000/v1.0/', False, None),
('http://mylocalhostname:5000/v1.0/', True, '<local>'),
('http://192.168.0.1/', False, ''),
),
)
def test_should_bypass_proxies_win_registry(url, expected, override, monkeypatch):
"""Tests for function should_bypass_proxies to check if proxy
can be bypassed or not with Windows registry settings
"""
@@ -634,6 +617,7 @@ def test_should_bypass_proxies_win_registry(url, expected, override,
import _winreg as winreg
class RegHandle:
def Close(self):
pass
@@ -646,6 +630,7 @@ def test_should_bypass_proxies_win_registry(url, expected, override,
if key is ie_settings:
if value_name == 'ProxyEnable':
return [1]
elif value_name == 'ProxyOverride':
return [override]
@@ -659,18 +644,19 @@ def test_should_bypass_proxies_win_registry(url, expected, override,
@pytest.mark.parametrize(
'env_name, value', (
('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain'),
('no_proxy', None),
('a_new_key', '192.168.0.0/24,127.0.0.1,localhost.localdomain'),
('a_new_key', None),
))
'env_name, value',
(
('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain'),
('no_proxy', None),
('a_new_key', '192.168.0.0/24,127.0.0.1,localhost.localdomain'),
('a_new_key', None),
),
)
def test_set_environ(env_name, value):
"""Tests set_environ will set environ values and will restore the environ."""
environ_copy = copy.deepcopy(os.environ)
with set_environ(env_name, value):
assert os.environ.get(env_name) == value
assert os.environ == environ_copy
tests/testserver/server.py
+16 -18
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
import threading
import socket
import select
@@ -8,7 +7,6 @@ import select
def consume_socket_content(sock, timeout=0.5):
chunks = 65536
content = b''
while True:
more_to_read = select.select([sock], [], [], timeout)[0]
if not more_to_read:
@@ -19,7 +17,6 @@ def consume_socket_content(sock, timeout=0.5):
break
content += new_content
return content
@@ -27,37 +24,38 @@ class Server(threading.Thread):
"""Dummy server using for unit testing"""
WAIT_EVENT_TIMEOUT = 5
def __init__(self, handler=None, host='localhost', port=0, requests_to_handle=1, wait_to_close_event=None):
def __init__(
self,
handler=None,
host='localhost',
port=0,
requests_to_handle=1,
wait_to_close_event=None,
):
super(Server, self).__init__()
self.handler = handler or consume_socket_content
self.handler_results = []
self.host = host
self.port = port
self.requests_to_handle = requests_to_handle
self.wait_to_close_event = wait_to_close_event
self.ready_event = threading.Event()
self.stop_event = threading.Event()
@classmethod
def text_response_server(cls, text, request_timeout=0.5, **kwargs):
def text_response_handler(sock):
request_content = consume_socket_content(sock, timeout=request_timeout)
sock.send(text.encode('utf-8'))
return request_content
return Server(text_response_handler, **kwargs)
@classmethod
def basic_response_server(cls, **kwargs):
return cls.text_response_server(
"HTTP/1.1 200 OK\r\n" +
"Content-Length: 0\r\n\r\n",
**kwargs
"HTTP/1.1 200 OK\r\n" + "Content-Length: 0\r\n\r\n", **kwargs
)
def run(self):
@@ -67,11 +65,10 @@ class Server(threading.Thread):
self.port = self.server_sock.getsockname()[1]
self.ready_event.set()
self._handle_requests()
if self.wait_to_close_event:
self.wait_to_close_event.wait(self.WAIT_EVENT_TIMEOUT)
finally:
self.ready_event.set() # just in case of exception
self.ready_event.set() # just in case of exception
self._close_server_sock_ignore_errors()
self.stop_event.set()
@@ -94,16 +91,18 @@ class Server(threading.Thread):
break
handler_result = self.handler(sock)
self.handler_results.append(handler_result)
def _accept_connection(self):
try:
ready, _, _ = select.select([self.server_sock], [], [], self.WAIT_EVENT_TIMEOUT)
ready, _, _ = select.select(
[self.server_sock], [], [], self.WAIT_EVENT_TIMEOUT
)
if not ready:
return None
return self.server_sock.accept()[0]
except (select.error, socket.error):
return None
@@ -120,8 +119,7 @@ class Server(threading.Thread):
# avoid server from waiting for event timeouts
# if an exception is found in the main thread
self.wait_to_close_event.set()
# ensure server thread doesn't get stuck waiting for connections
self._close_server_sock_ignore_errors()
self.join()
return False # allow exceptions to propagate
return False # allow exceptions to propagate
tests/utils.py
+1 -1
View File
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
import contextlib
import os
@@ -14,6 +13,7 @@ def override_environ(**kwargs):
os.environ[key] = value
try:
yield
finally:
os.environ.clear()
os.environ.update(save_env)